Add oe.which() function.
[bitbake.git] / bin / oe / __init__.py
1 #!/usr/bin/python
2 """
3 OpenEmbedded Build System Python Library
4
5 Copyright: (c) 2003 by Holger Schurig
6
7 Part of this code has been shamelessly stolen from Gentoo's portage.py.
8 This source had GPL-2 as license, so the same goes for this file.
9
10 Please visit http://www.openembedded.org/phpwiki/ for more info.
11
12 Try "pydoc ./oe.py" to get some nice output.
13 """
14
15 __version__ = "1.0"
16
17 __all__ = [
18
19         "debug",
20         "note",
21         "error",
22         "fatal",
23
24         "mkdirhier",
25         "movefile",
26
27         "tokenize",
28         "evaluate",
29         "flatten",
30         "relparse",
31         "ververify",
32         "isjustname",
33         "isspecific",
34         "pkgsplit",
35         "catpkgsplit",
36         "vercmp",
37         "pkgcmp",
38         "dep_parenreduce",
39         "dep_opconvert",
40         "digraph",
41
42 # fetch
43         "decodeurl",
44         "encodeurl",
45
46 # file parsing
47         "read_config",
48         "read_oe",
49
50 # data handling
51         "inherit_os_env",
52         "getenv",
53         "setenv",
54         "set_automatic_vars",
55         "set_additional_vars",
56         "update_env",
57         "env",
58         "envflags",
59         "expand",
60         "print_missing_env",
61         "print_orphan_env",
62
63 # modules
64         "parse",
65         "data",
66         "event",
67         "build",
68         "fetch",
69  ]
70
71 import sys,os,string,types,re
72
73 #projectdir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
74 projectdir = os.getcwd()
75 env = {}
76
77 class VarExpandError(Exception):
78         pass
79
80 class MalformedUrl(Exception):
81         """Exception raised when encountering an invalid url"""
82
83
84 #######################################################################
85 #######################################################################
86 #
87 # SECTION: Debug
88 #
89 # PURPOSE: little functions to make yourself known
90 #
91 #######################################################################
92 #######################################################################
93
94 debug_prepend = ''
95
96
97 def debug(lvl, *args):
98         if env.has_key('OEDEBUG') and (env['OEDEBUG'] >= str(lvl)):
99                 print debug_prepend + 'DEBUG:', string.join(args, '')
100
101 def note(*args):
102         print debug_prepend + 'NOTE:', string.join(args, '')
103
104 def error(*args):
105         print debug_prepend + 'ERROR:', string.join(args, '')
106
107 def fatal(*args):
108         print debug_prepend + 'ERROR:', string.join(args, '')
109         sys.exit(1)
110
111
112 #######################################################################
113 #######################################################################
114 #
115 # SECTION: File
116 #
117 # PURPOSE: Basic file and directory tree related functions
118 #
119 #######################################################################
120 #######################################################################
121
122 def mkdirhier(dir):
123         """Create a directory like 'mkdir -p', but does not complain if
124         directory already exists like os.makedirs
125         """
126
127         debug(3, "mkdirhier(%s)" % dir)
128         try:
129                 os.makedirs(dir)
130                 debug(2, "created " + dir)
131         except OSError, e:
132                 if e.errno != 17: raise e
133
134
135 #######################################################################
136
137 import stat
138
139 def movefile(src,dest,newmtime=None,sstat=None):
140         """Moves a file from src to dest, preserving all permissions and
141         attributes; mtime will be preserved even when moving across
142         filesystems.  Returns true on success and false on failure. Move is
143         atomic.
144         """
145
146         #print "movefile("+src+","+dest+","+str(newmtime)+","+str(sstat)+")"
147         try:
148                 if not sstat:
149                         sstat=os.lstat(src)
150         except Exception, e:
151                 print "!!! Stating source file failed... movefile()"
152                 print "!!!",e
153                 return None
154
155         destexists=1
156         try:
157                 dstat=os.lstat(dest)
158         except:
159                 dstat=os.lstat(os.path.dirname(dest))
160                 destexists=0
161
162         if destexists:
163                 if stat.S_ISLNK(dstat[stat.ST_MODE]):
164                         try:
165                                 os.unlink(dest)
166                                 destexists=0
167                         except Exception, e:
168                                 pass
169
170         if stat.S_ISLNK(sstat[stat.ST_MODE]):
171                 try:
172                         target=os.readlink(src)
173                         if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
174                                 os.unlink(dest)
175                         os.symlink(target,dest)
176 #                       os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
177                         return os.lstat(dest)
178                 except Exception, e:
179                         print "!!! failed to properly create symlink:"
180                         print "!!!",dest,"->",target
181                         print "!!!",e
182                         return None
183
184         renamefailed=1
185         if sstat[stat.ST_DEV]==dstat[stat.ST_DEV]:
186                 try:
187                         ret=os.rename(src,dest)
188                         renamefailed=0
189                 except Exception, e:
190                         import errno
191                         if e[0]!=errno.EXDEV:
192                                 # Some random error.
193                                 print "!!! Failed to move",src,"to",dest
194                                 print "!!!",e
195                                 return None
196                         # Invalid cross-device-link 'bind' mounted or actually Cross-Device
197
198         if renamefailed:
199                 didcopy=0
200                 if stat.S_ISREG(sstat[stat.ST_MODE]):
201                         try: # For safety copy then move it over.
202                                 shutil.copyfile(src,dest+"#new")
203                                 os.rename(dest+"#new",dest)
204                                 didcopy=1
205                         except Exception, e:
206                                 print '!!! copy',src,'->',dest,'failed.'
207                                 print "!!!",e
208                                 return None
209                 else:
210                         #we don't yet handle special, so we need to fall back to /bin/mv
211                         a=getstatusoutput("/bin/mv -f "+"'"+src+"' '"+dest+"'")
212                         if a[0]!=0:
213                                 print "!!! Failed to move special file:"
214                                 print "!!! '"+src+"' to '"+dest+"'"
215                                 print "!!!",a
216                                 return None # failure
217                 try:
218                         if didcopy:
219                                 missingos.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
220                                 os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
221                                 os.unlink(src)
222                 except Exception, e:
223                         print "!!! Failed to chown/chmod/unlink in movefile()"
224                         print "!!!",dest
225                         print "!!!",e
226                         return None
227
228         if newmtime:
229                 os.utime(dest,(newmtime,newmtime))
230         else:
231                 os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
232                 newmtime=sstat[stat.ST_MTIME]
233         return newmtime
234
235
236
237 #######################################################################
238 #######################################################################
239 #
240 # SECTION: Download
241 #
242 # PURPOSE: Download via HTTP, FTP, CVS, BITKEEPER, handling of MD5-signatures
243 #          and mirrors
244 #
245 #######################################################################
246 #######################################################################
247
248 def decodeurl(url):
249         """Decodes an URL into the tokens (scheme, network location, path,
250         user, password, parameters). 
251
252         >>> decodeurl("http://www.google.com/index.html")
253         ('http', 'www.google.com', '/index.html', '', '', {})
254
255         CVS url with username, host and cvsroot. The cvs module to check out is in the
256         parameters:
257
258         >>> decodeurl("cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg")
259         ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'})
260
261         Dito, but this time the username has a password part. And we also request a special tag
262         to check out.
263
264         >>> decodeurl("cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;module=familiar/dist/ipkg;tag=V0-99-81")
265         ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'})
266         """
267
268         #debug(3, "decodeurl('%s')" % url)
269         m = re.compile('([^:]*):/*(.+@)?([^/]+)(/[^;]+);?(.*)').match(url)
270         if not m:
271                 raise MalformedUrl(url)
272
273         type = m.group(1)
274         host = m.group(3)
275         path = m.group(4)
276         user = m.group(2)
277         parm = m.group(5)
278         #print "type:", type
279         #print "host:", host
280         #print "path:", path
281         #print "parm:", parm
282         if user:
283                 m = re.compile('([^:]+)(:?(.*))@').match(user)
284                 if m:
285                         user = m.group(1)
286                         pswd = m.group(3)
287         else:
288                 user = ''
289                 pswd = ''
290         #print "user:", user
291         #print "pswd:", pswd
292         #print
293         p = {}
294         if parm:
295                 for s in parm.split(';'):
296                         s1,s2 = s.split('=')
297                         p[s1] = s2
298                         
299         return (type, host, path, user, pswd, p)
300                 
301 #######################################################################
302
303 def encodeurl(decoded):
304         """Encodes a URL from tokens (scheme, network location, path,
305         user, password, parameters). 
306
307         >>> encodeurl(['http', 'www.google.com', '/index.html', '', '', {}])
308
309         "http://www.google.com/index.html"
310
311         CVS with username, host and cvsroot. The cvs module to check out is in the
312         parameters:
313
314         >>> encodeurl(['cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}])
315
316         "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg"
317
318         Dito, but this time the username has a password part. And we also request a special tag
319         to check out.
320
321         >>> encodeurl(['cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'}])
322
323         "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;module=familiar/dist/ipkg;tag=V0-99-81"
324         """
325
326         (type, host, path, user, pswd, p) = decoded
327
328         if not type or not host or not path:
329                 fatal("invalid or missing parameters for url encoding")
330
331         url = '%s://' % type
332         if user:
333                 url += "%s" % user
334                 if pswd:
335                         url += ":%s" % pswd
336                 url += "@"
337         url += "%s%s" % (host, path)
338         if p:
339                 for parm in p.keys():
340                         url += ";%s=%s" % (parm, p[parm])
341
342         return url
343
344 #######################################################################
345
346 def which(path, item, direction = 1):
347         """Useful function for locating a file in a PATH"""
348         found = item
349         for p in string.split(path or "", ":"):
350                 if os.path.exists(os.path.join(p, item)):
351                         found = os.path.join(p, item)
352                         if direction == 0:
353                                 break
354         return found
355
356 #######################################################################
357
358
359
360
361 #######################################################################
362 #######################################################################
363 #
364 # SECTION: Dependency
365 #
366 # PURPOSE: Compare build & run dependencies
367 #
368 #######################################################################
369 #######################################################################
370
371 def tokenize(mystring):
372         """Breaks a string like 'foo? (bar) oni? (blah (blah))' into (possibly embedded) lists:
373
374         >>> tokenize("x")
375         ['x']
376         >>> tokenize("x y")
377         ['x', 'y']
378         >>> tokenize("(x y)")
379         [['x', 'y']]
380         >>> tokenize("(x y) b c")
381         [['x', 'y'], 'b', 'c']
382         >>> tokenize("foo? (bar) oni? (blah (blah))")
383         ['foo?', ['bar'], 'oni?', ['blah', ['blah']]]
384         >>> tokenize("sys-apps/linux-headers nls? (sys-devel/gettext)")
385         ['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']]
386         """
387
388         newtokens = []
389         curlist   = newtokens
390         prevlists = []
391         level     = 0
392         accum     = ""
393         for x in mystring:
394                 if x=="(":
395                         if accum:
396                                 curlist.append(accum)
397                                 accum=""
398                         prevlists.append(curlist)
399                         curlist=[]
400                         level=level+1
401                 elif x==")":
402                         if accum:
403                                 curlist.append(accum)
404                                 accum=""
405                         if level==0:
406                                 print "!!! tokenizer: Unmatched left parenthesis in:\n'"+mystring+"'"
407                                 return None
408                         newlist=curlist
409                         curlist=prevlists.pop()
410                         curlist.append(newlist)
411                         level=level-1
412                 elif x in string.whitespace:
413                         if accum:
414                                 curlist.append(accum)
415                                 accum=""
416                 else:
417                         accum=accum+x
418         if accum:
419                 curlist.append(accum)
420         if (level!=0):
421                 print "!!! tokenizer: Exiting with unterminated parenthesis in:\n'"+mystring+"'"
422                 return None
423         return newtokens
424
425
426 #######################################################################
427
428 def evaluate(tokens,mydefines,allon=0):
429         """Removes tokens based on whether conditional definitions exist or not.
430         Recognizes !
431
432         >>> evaluate(['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']], {})
433         ['sys-apps/linux-headers']
434
435         Negate the flag:
436
437         >>> evaluate(['sys-apps/linux-headers', '!nls?', ['sys-devel/gettext']], {})
438         ['sys-apps/linux-headers', ['sys-devel/gettext']]
439
440         Define 'nls':
441
442         >>> evaluate(['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']], {"nls":1})
443         ['sys-apps/linux-headers', ['sys-devel/gettext']]
444
445         Turn allon on:
446
447         >>> evaluate(['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']], {}, True)
448         ['sys-apps/linux-headers', ['sys-devel/gettext']]
449         """
450
451         if tokens == None:
452                 return None
453         mytokens = tokens + []          # this copies the list
454         pos = 0
455         while pos < len(mytokens):
456                 if type(mytokens[pos]) == types.ListType:
457                         evaluate(mytokens[pos], mydefines)
458                         if not len(mytokens[pos]):
459                                 del mytokens[pos]
460                                 continue
461                 elif mytokens[pos][-1] == "?":
462                         cur = mytokens[pos][:-1]
463                         del mytokens[pos]
464                         if allon:
465                                 if cur[0] == "!":
466                                         del mytokens[pos]
467                         else:
468                                 if cur[0] == "!":
469                                         if (cur[1:] in mydefines) and (pos < len(mytokens)):
470                                                 del mytokens[pos]
471                                                 continue
472                                 elif (cur not in mydefines) and (pos < len(mytokens)):
473                                         del mytokens[pos]
474                                         continue
475                 pos = pos + 1
476         return mytokens
477
478
479 #######################################################################
480
481 def flatten(mytokens):
482         """Converts nested arrays into a flat arrays:
483
484         >>> flatten([1,[2,3]])
485         [1, 2, 3]
486         >>> flatten(['sys-apps/linux-headers', ['sys-devel/gettext']])
487         ['sys-apps/linux-headers', 'sys-devel/gettext']
488         """
489
490         newlist=[]
491         for x in mytokens:
492                 if type(x)==types.ListType:
493                         newlist.extend(flatten(x))
494                 else:
495                         newlist.append(x)
496         return newlist
497
498
499 #######################################################################
500
501 _package_weights_ = {"pre":-2,"p":0,"alpha":-4,"beta":-3,"rc":-1}       # dicts are unordered
502 _package_ends_    = ["pre", "p", "alpha", "beta", "rc", "cvs", "bk", "HEAD" ]                   # so we need ordered list
503
504 def relparse(myver):
505         """Parses the last elements of a version number into a triplet, that can
506         later be compared:
507
508         >>> relparse('1.2_pre3')
509         [1.2, -2, 3.0]
510         >>> relparse('1.2b')
511         [1.2, 98, 0]
512         >>> relparse('1.2')
513         [1.2, 0, 0]
514         """
515
516         number   = 0
517         p1       = 0
518         p2       = 0
519         mynewver = string.split(myver,"_")
520         if len(mynewver)==2:
521                 # an _package_weights_
522                 number = string.atof(mynewver[0])
523                 match = 0
524                 for x in _package_ends_:
525                         elen = len(x)
526                         if mynewver[1][:elen] == x:
527                                 match = 1
528                                 p1 = _package_weights_[x]
529                                 try:
530                                         p2 = string.atof(mynewver[1][elen:])
531                                 except:
532                                         p2 = 0
533                                 break
534                 if not match:   
535                         # normal number or number with letter at end
536                         divider = len(myver)-1
537                         if myver[divider:] not in "1234567890":
538                                 # letter at end
539                                 p1 = ord(myver[divider:])
540                                 number = string.atof(myver[0:divider])
541                         else:
542                                 number = string.atof(myver)             
543         else:
544                 # normal number or number with letter at end
545                 divider = len(myver)-1
546                 if myver[divider:] not in "1234567890":
547                         #letter at end
548                         p1     = ord(myver[divider:])
549                         number = string.atof(myver[0:divider])
550                 else:
551                         number = string.atof(myver)  
552         return [number,p1,p2]
553
554
555 #######################################################################
556
557 __ververify_cache__ = {}
558
559 def ververify(myorigval,silent=1):
560         """Returns 1 if given a valid version string, els 0. Valid versions are in the format
561
562         <v1>.<v2>...<vx>[a-z,_{_package_weights_}[vy]]
563
564         >>> ververify('2.4.20')
565         1
566         >>> ververify('2.4..20')                # two dots
567         0
568         >>> ververify('2.x.20')                 # 'x' is not numeric
569         0
570         >>> ververify('2.4.20a')
571         1
572         >>> ververify('2.4.20cvs')              # only one trailing letter
573         0
574         >>> ververify('1a')
575         1
576         >>> ververify('test_a')                 # no version at all
577         0
578         >>> ververify('2.4.20_beta1')
579         1
580         >>> ververify('2.4.20_beta')
581         1
582         >>> ververify('2.4.20_wrongext')        # _wrongext is no valid trailer
583         0
584         """
585
586         # Lookup the cache first
587         try:
588                 return __ververify_cache__[myorigval]
589         except KeyError:
590                 pass
591
592         if len(myorigval) == 0:
593                 if not silent:
594                         error("package version is empty")
595                 __ververify_cache__[myorigval] = 0
596                 return 0
597         myval = string.split(myorigval,'.')
598         if len(myval)==0:
599                 if not silent:
600                         error("package name has empty version string")
601                 __ververify_cache__[myorigval] = 0
602                 return 0
603         # all but the last version must be a numeric
604         for x in myval[:-1]:
605                 if not len(x):
606                         if not silent:
607                                 error("package version has two points in a row")
608                         __ververify_cache__[myorigval] = 0
609                         return 0
610                 try:
611                         foo = string.atoi(x)
612                 except:
613                         if not silent:
614                                 error("package version contains non-numeric '"+x+"'")
615                         __ververify_cache__[myorigval] = 0
616                         return 0
617         if not len(myval[-1]):
618                         if not silent:
619                                 error("package version has trailing dot")
620                         __ververify_cache__[myorigval] = 0
621                         return 0
622         try:
623                 foo = string.atoi(myval[-1])
624                 __ververify_cache__[myorigval] = 1
625                 return 1
626         except:
627                 pass
628
629         # ok, our last component is not a plain number or blank, let's continue
630         if myval[-1][-1] in string.lowercase:
631                 try:
632                         foo = string.atoi(myval[-1][:-1])
633                         return 1
634                         __ververify_cache__[myorigval] = 1
635                         # 1a, 2.0b, etc.
636                 except:
637                         pass
638         # ok, maybe we have a 1_alpha or 1_beta2; let's see
639         ep=string.split(myval[-1],"_")
640         if len(ep)!= 2:
641                 if not silent:
642                         error("package version has more than one letter at then end")
643                 __ververify_cache__[myorigval] = 0
644                 return 0
645         try:
646                 foo = string.atoi(ep[0])
647         except:
648                 # this needs to be numeric, i.e. the "1" in "1_alpha"
649                 if not silent:
650                         error("package version must have numeric part before the '_'")
651                 __ververify_cache__[myorigval] = 0
652                 return 0
653
654         for mye in _package_ends_:
655                 if ep[1][0:len(mye)] == mye:
656                         if len(mye) == len(ep[1]):
657                                 # no trailing numeric is ok
658                                 __ververify_cache__[myorigval] = 1
659                                 return 1
660                         else:
661                                 try:
662                                         foo = string.atoi(ep[1][len(mye):])
663                                         __ververify_cache__[myorigval] = 1
664                                         return 1
665                                 except:
666                                         # if no _package_weights_ work, *then* we return 0
667                                         pass    
668         if not silent:
669                 error("package version extension after '_' is invalid")
670         __ververify_cache__[myorigval] = 0
671         return 0
672
673
674 def isjustname(mypkg):
675         myparts = string.split(mypkg,'-')
676         for x in myparts:
677                 if ververify(x):
678                         return 0
679         return 1
680
681
682 _isspecific_cache_={}
683
684 def isspecific(mypkg):
685         "now supports packages with no category"
686         try:
687                 return __isspecific_cache__[mypkg]
688         except:
689                 pass
690
691         mysplit = string.split(mypkg,"/")
692         if not isjustname(mysplit[-1]):
693                         __isspecific_cache__[mypkg] = 1
694                         return 1
695         __isspecific_cache__[mypkg] = 0
696         return 0
697
698
699 #######################################################################
700
701 __pkgsplit_cache__={}
702
703 def pkgsplit(mypkg, silent=1):
704
705         """This function can be used as a package verification function. If
706         it is a valid name, pkgsplit will return a list containing:
707         [pkgname, pkgversion(norev), pkgrev ].
708
709         >>> pkgsplit('')
710         >>> pkgsplit('x')
711         >>> pkgsplit('x-')
712         >>> pkgsplit('-1')
713         >>> pkgsplit('glibc-1.2-8.9-r7')
714         >>> pkgsplit('glibc-2.2.5-r7')
715         ['glibc', '2.2.5', 'r7']
716         >>> pkgsplit('foo-1.2-1')
717         >>> pkgsplit('Mesa-3.0')
718         ['Mesa', '3.0', 'r0']
719         """
720
721         try:
722                 return __pkgsplit_cache__[mypkg]
723         except KeyError:
724                 pass
725
726         myparts = string.split(mypkg,'-')
727         if len(myparts) < 2:
728                 if not silent:
729                         error("package name without name or version part")
730                 __pkgsplit_cache__[mypkg] = None
731                 return None
732         for x in myparts:
733                 if len(x) == 0:
734                         if not silent:
735                                 error("package name with empty name or version part")
736                         __pkgsplit_cache__[mypkg] = None
737                         return None
738         # verify rev
739         revok = 0
740         myrev = myparts[-1]
741         ververify(myrev, 0)
742         if len(myrev) and myrev[0] == "r":
743                 try:
744                         string.atoi(myrev[1:])
745                         revok = 1
746                 except: 
747                         pass
748         if revok:
749                 if ververify(myparts[-2]):
750                         if len(myparts) == 2:
751                                 __pkgsplit_cache__[mypkg] = None
752                                 return None
753                         else:
754                                 for x in myparts[:-2]:
755                                         if ververify(x):
756                                                 __pkgsplit_cache__[mypkg]=None
757                                                 return None
758                                                 # names can't have versiony looking parts
759                                 myval=[string.join(myparts[:-2],"-"),myparts[-2],myparts[-1]]
760                                 __pkgsplit_cache__[mypkg]=myval
761                                 return myval
762                 else:
763                         __pkgsplit_cache__[mypkg] = None
764                         return None
765
766         elif ververify(myparts[-1],silent):
767                 if len(myparts)==1:
768                         if not silent:
769                                 print "!!! Name error in",mypkg+": missing name part."
770                         __pkgsplit_cache__[mypkg]=None
771                         return None
772                 else:
773                         for x in myparts[:-1]:
774                                 if ververify(x):
775                                         if not silent: error("package name has multiple version parts")
776                                         __pkgsplit_cache__[mypkg] = None
777                                         return None
778                         myval = [string.join(myparts[:-1],"-"), myparts[-1],"r0"]
779                         __pkgsplit_cache__[mypkg] = myval
780                         return myval
781         else:
782                 __pkgsplit_cache__[mypkg] = None
783                 return None
784
785
786 #######################################################################
787
788 __catpkgsplit_cache__ = {}
789
790 def catpkgsplit(mydata,silent=1):
791         """returns [cat, pkgname, version, rev ]
792
793         >>> catpkgsplit('sys-libs/glibc-1.2-r7')
794         ['sys-libs', 'glibc', '1.2', 'r7']
795         >>> catpkgsplit('glibc-1.2-r7')
796         ['null', 'glibc', '1.2', 'r7']
797         """
798
799         try:
800                 return __catpkgsplit_cache__[mydata]
801         except KeyError:
802                 pass
803
804         cat = os.path.basename(os.path.dirname(mydata))
805         mydata = os.path.join(cat, os.path.basename(mydata))
806 #       if mydata[:len(projectdir)] == projectdir:
807 #               mydata = mydata[len(projectdir)+1:]
808         if mydata[-3:] == '.oe':
809                 mydata = mydata[:-3]
810
811         mysplit = mydata.split("/")
812         p_split = None
813         splitlen = len(mysplit)
814         if splitlen == 1:
815                 retval = [None]
816                 p_split = pkgsplit(mydata,silent)
817         else:
818                 retval = [mysplit[splitlen - 2]]
819                 p_split = pkgsplit(mysplit[splitlen - 1],silent)
820         if not p_split:
821                 __catpkgsplit_cache__[mydata] = None
822                 return None
823         retval.extend(p_split)
824         __catpkgsplit_cache__[mydata] = retval
825         return retval
826
827
828 #######################################################################
829
830 __vercmp_cache__ = {}
831
832 def vercmp(val1,val2):
833         """This takes two version strings and returns an integer to tell you whether
834         the versions are the same, val1>val2 or val2>val1.
835         
836         >>> vercmp('1', '2')
837         -1.0
838         >>> vercmp('2', '1')
839         1.0
840         >>> vercmp('1', '1.0')
841         0
842         >>> vercmp('1', '1.1')
843         -1.0
844         >>> vercmp('1.1', '1_p2')
845         1.0
846         """
847
848         # quick short-circuit
849         if val1 == val2:
850                 return 0
851         valkey = val1+" "+val2
852
853         # cache lookup
854         try:
855                 return __vercmp_cache__[valkey]
856                 try:
857                         return - __vercmp_cache__[val2+" "+val1]
858                 except KeyError:
859                         pass
860         except KeyError:
861                 pass
862         
863         # consider 1_p2 vc 1.1
864         # after expansion will become (1_p2,0) vc (1,1)
865         # then 1_p2 is compared with 1 before 0 is compared with 1
866         # to solve the bug we need to convert it to (1,0_p2)
867         # by splitting _prepart part and adding it back _after_expansion
868
869         val1_prepart = val2_prepart = ''
870         if val1.count('_'):
871                 val1, val1_prepart = val1.split('_', 1)
872         if val2.count('_'):
873                 val2, val2_prepart = val2.split('_', 1)
874
875         # replace '-' by '.'
876         # FIXME: Is it needed? can val1/2 contain '-'?
877
878         val1 = string.split(val1,'-')
879         if len(val1) == 2:
880                 val1[0] = val1[0] +"."+ val1[1]
881         val2 = string.split(val2,'-')
882         if len(val2) == 2:
883                 val2[0] = val2[0] +"."+ val2[1]
884
885         val1 = string.split(val1[0],'.')
886         val2 = string.split(val2[0],'.')
887
888         # add back decimal point so that .03 does not become "3" !
889         for x in range(1,len(val1)):
890                 if val1[x][0] == '0' :
891                         val1[x] = '.' + val1[x]
892         for x in range(1,len(val2)):
893                 if val2[x][0] == '0' :
894                         val2[x] = '.' + val2[x]
895
896         # extend varion numbers
897         if len(val2) < len(val1):
898                 val2.extend(["0"]*(len(val1)-len(val2)))
899         elif len(val1) < len(val2):
900                 val1.extend(["0"]*(len(val2)-len(val1)))
901
902         # add back _prepart tails
903         if val1_prepart:
904                 val1[-1] += '_' + val1_prepart
905         if val2_prepart:
906                 val2[-1] += '_' + val2_prepart
907         # The above code will extend version numbers out so they
908         # have the same number of digits.
909         for x in range(0,len(val1)):
910                 cmp1 = relparse(val1[x])
911                 cmp2 = relparse(val2[x])
912                 for y in range(0,3):
913                         myret = cmp1[y] - cmp2[y]
914                         if myret != 0:
915                                 __vercmp_cache__[valkey] = myret
916                                 return myret
917         __vercmp_cache__[valkey] = 0
918         return 0
919
920
921 #######################################################################
922
923 def pkgcmp(pkg1,pkg2):
924         """ Compares two packages, which should have been split via
925         pkgsplit(). if the return value val is less than zero, then pkg2 is
926         newer than pkg1, zero if equal and positive if older.
927
928         >>> pkgcmp(['glibc', '2.2.5', 'r7'], ['glibc', '2.2.5', 'r7'])
929         0
930         >>> pkgcmp(['glibc', '2.2.5', 'r4'], ['glibc', '2.2.5', 'r7'])
931         -1
932         >>> pkgcmp(['glibc', '2.2.5', 'r7'], ['glibc', '2.2.5', 'r2'])
933         1
934         """
935         
936         mycmp = vercmp(pkg1[1],pkg2[1])
937         if mycmp > 0:
938                 return 1
939         if mycmp < 0:
940                 return -1
941         r1=string.atoi(pkg1[2][1:])
942         r2=string.atoi(pkg2[2][1:])
943         if r1 > r2:
944                 return 1
945         if r2 > r1:
946                 return -1
947         return 0
948
949
950 #######################################################################
951
952 def dep_parenreduce(mysplit, mypos=0):
953         """Accepts a list of strings, and converts '(' and ')' surrounded items to sub-lists:
954
955         >>> dep_parenreduce([''])
956         ['']
957         >>> dep_parenreduce(['1', '2', '3'])
958         ['1', '2', '3']
959         >>> dep_parenreduce(['1', '(', '2', '3', ')', '4'])
960         ['1', ['2', '3'], '4']
961         """
962
963         while mypos < len(mysplit): 
964                 if mysplit[mypos] == "(":
965                         firstpos = mypos
966                         mypos = mypos + 1
967                         while mypos < len(mysplit):
968                                 if mysplit[mypos] == ")":
969                                         mysplit[firstpos:mypos+1] = [mysplit[firstpos+1:mypos]]
970                                         mypos = firstpos
971                                         break
972                                 elif mysplit[mypos] == "(":
973                                         # recurse
974                                         mysplit = dep_parenreduce(mysplit,mypos)
975                                 mypos = mypos + 1
976                 mypos = mypos + 1
977         return mysplit
978
979
980 def dep_opconvert(mysplit, myuse):
981         "Does dependency operator conversion"
982         
983         mypos   = 0
984         newsplit = []
985         while mypos < len(mysplit):
986                 if type(mysplit[mypos]) == types.ListType:
987                         newsplit.append(dep_opconvert(mysplit[mypos],myuse))
988                         mypos += 1
989                 elif mysplit[mypos] == ")":
990                         # mismatched paren, error
991                         return None
992                 elif mysplit[mypos]=="||":
993                         if ((mypos+1)>=len(mysplit)) or (type(mysplit[mypos+1])!=types.ListType):
994                                 # || must be followed by paren'd list
995                                 return None
996                         try:
997                                 mynew = dep_opconvert(mysplit[mypos+1],myuse)
998                         except Exception, e:
999                                 error("unable to satisfy OR dependancy: " + string.join(mysplit," || "))
1000                                 raise e
1001                         mynew[0:0] = ["||"]
1002                         newsplit.append(mynew)
1003                         mypos += 2
1004                 elif mysplit[mypos][-1] == "?":
1005                         # use clause, i.e "gnome? ( foo bar )"
1006                         # this is a quick and dirty hack so that repoman can enable all USE vars:
1007                         if (len(myuse) == 1) and (myuse[0] == "*"):
1008                                 # enable it even if it's ! (for repoman) but kill it if it's
1009                                 # an arch variable that isn't for this arch. XXX Sparc64?
1010                                 if (mysplit[mypos][:-1] not in settings.usemask) or \
1011                                                 (mysplit[mypos][:-1]==settings["ARCH"]):
1012                                         enabled=1
1013                                 else:
1014                                         enabled=0
1015                         else:
1016                                 if mysplit[mypos][0] == "!":
1017                                         myusevar = mysplit[mypos][1:-1]
1018                                         enabled = not myusevar in myuse
1019                                         #if myusevar in myuse:
1020                                         #       enabled = 0
1021                                         #else:
1022                                         #       enabled = 1
1023                                 else:
1024                                         myusevar=mysplit[mypos][:-1]
1025                                         enabled = myusevar in myuse
1026                                         #if myusevar in myuse:
1027                                         #       enabled=1
1028                                         #else:
1029                                         #       enabled=0
1030                         if (mypos +2 < len(mysplit)) and (mysplit[mypos+2] == ":"):
1031                                 # colon mode
1032                                 if enabled:
1033                                         # choose the first option
1034                                         if type(mysplit[mypos+1]) == types.ListType:
1035                                                 newsplit.append(dep_opconvert(mysplit[mypos+1],myuse))
1036                                         else:
1037                                                 newsplit.append(mysplit[mypos+1])
1038                                 else:
1039                                         # choose the alternate option
1040                                         if type(mysplit[mypos+1]) == types.ListType:
1041                                                 newsplit.append(dep_opconvert(mysplit[mypos+3],myuse))
1042                                         else:
1043                                                 newsplit.append(mysplit[mypos+3])
1044                                 mypos += 4
1045                         else:
1046                                 # normal use mode
1047                                 if enabled:
1048                                         if type(mysplit[mypos+1]) == types.ListType:
1049                                                 newsplit.append(dep_opconvert(mysplit[mypos+1],myuse))
1050                                         else:
1051                                                 newsplit.append(mysplit[mypos+1])
1052                                 # otherwise, continue
1053                                 mypos += 2
1054                 else:
1055                         # normal item
1056                         newsplit.append(mysplit[mypos])
1057                         mypos += 1
1058         return newsplit
1059
1060 class digraph:
1061         """beautiful directed graph object"""
1062
1063         def __init__(self):
1064                 self.dict={}
1065                 #okeys = keys, in order they were added (to optimize firstzero() ordering)
1066                 self.okeys=[]
1067
1068         def __str__(self):
1069                 str = ""
1070                 for key in self.okeys:
1071                         str += "%s:\t%s\n" % (key, self.dict[key][1])
1072                 return str
1073
1074         def addnode(self,mykey,myparent):
1075                 if not self.dict.has_key(mykey):
1076                         self.okeys.append(mykey)
1077                         if myparent==None:
1078                                 self.dict[mykey]=[0,[]]
1079                         else:
1080                                 self.dict[mykey]=[0,[myparent]]
1081                                 self.dict[myparent][0]=self.dict[myparent][0]+1
1082                         return
1083                 if myparent and (not myparent in self.dict[mykey][1]):
1084                         self.dict[mykey][1].append(myparent)
1085                         self.dict[myparent][0]=self.dict[myparent][0]+1
1086         
1087         def delnode(self,mykey, ref = 1):
1088                 """Delete a node
1089
1090                 If ref is 1, remove references to this node from other nodes.
1091                 If ref is 2, remove nodes that reference this node."""
1092                 if not self.dict.has_key(mykey):
1093                         return
1094                 for x in self.dict[mykey][1]:
1095                         self.dict[x][0]=self.dict[x][0]-1
1096                 del self.dict[mykey]
1097                 while 1:
1098                         try:
1099                                 self.okeys.remove(mykey)        
1100                         except ValueError:
1101                                 break
1102                 if ref:
1103                         __kill = []
1104                         for k in self.okeys:
1105                                 if mykey in self.dict[k][1]:
1106                                         if ref == 1 or ref == 2:
1107                                                 self.dict[k][1].remove(mykey)
1108                                         if ref == 2:
1109                                                 __kill.append(k)
1110                         for l in __kill:
1111                                 self.delnode(l, ref)
1112         
1113         def allnodes(self):
1114                 "returns all nodes in the dictionary"
1115                 return self.dict.keys()
1116         
1117         def firstzero(self):
1118                 "returns first node with zero references, or NULL if no such node exists"
1119                 for x in self.okeys:
1120                         if self.dict[x][0]==0:
1121                                 return x
1122                 return None 
1123
1124         def firstnonzero(self):
1125                 "returns first node with nonzero references, or NULL if no such node exists"
1126                 for x in self.okeys:
1127                         if self.dict[x][0]!=0:
1128                                 return x
1129                 return None 
1130
1131
1132         def allzeros(self):
1133                 "returns all nodes with zero references, or NULL if no such node exists"
1134                 zerolist = []
1135                 for x in self.dict.keys():
1136                         if self.dict[x][0]==0:
1137                                 zerolist.append(x)
1138                 return zerolist
1139
1140         def hasallzeros(self):
1141                 "returns 0/1, Are all nodes zeros? 1 : 0"
1142                 zerolist = []
1143                 for x in self.dict.keys():
1144                         if self.dict[x][0]!=0:
1145                                 return 0
1146                 return 1
1147
1148         def empty(self):
1149                 if len(self.dict)==0:
1150                         return 1
1151                 return 0
1152
1153         def hasnode(self,mynode):
1154                 return self.dict.has_key(mynode)
1155
1156         def getparents(self, item):
1157                 if not self.hasnode(item):
1158                         return []
1159                 return self.dict[item][1]
1160         
1161         def getchildren(self, item):
1162                 if not self.hasnode(item):
1163                         return []
1164                 children = [i for i in self.okeys if item in self.getparents(i)]
1165                 return children
1166         
1167         def walkdown(self, item, callback):
1168                 __down_callback_cache = []
1169                 __recurse_count = 0
1170                 if not self.hasnode(item):
1171                         return 0
1172         
1173                 if __down_callback_cache.count(item):
1174                         return 1
1175
1176                 parents = self.getparents(item)
1177                 children = self.getchildren(item)
1178                 for p in parents:
1179                         if p in children:
1180                                 print "%s is both parent and child of %s, aborting" % (p, item)
1181                                 return 0
1182                         if item == p:
1183                                 print "eek, i'm my own parent!"
1184                                 return 0
1185 #                       print "walkdown => item: %s, p: %s" % (item, p)
1186                         ret = self.walkdown(p, callback)
1187                         if ret == 0:
1188                                 return 0
1189
1190                 __down_callback_cache.append(item)
1191                 return callback(self, item)
1192         
1193         def walkup(self, item, callback):
1194                 if not self.hasnode(item):
1195                         return 0
1196         
1197                 children = self.getchildren(item)
1198                 for c in children:
1199                         ret = self.walkup(c, callback)
1200                         if ret == 0:
1201                                 return 0
1202                 return callback(self, item)
1203
1204         def copy(self):
1205                 mygraph=digraph()
1206                 for x in self.dict.keys():
1207                         mygraph.dict[x]=self.dict[x][:]
1208                         mygraph.okeys=self.okeys[:]
1209                 return mygraph
1210
1211 #######################################################################
1212 #######################################################################
1213 #
1214 # SECTION: Config
1215 #
1216 # PURPOSE: Reading and handling of system/target-specific/local configuration
1217 #          reading of package configuration
1218 #
1219 #######################################################################
1220 #######################################################################
1221
1222 def reader(cfgfile, feeder):
1223         """Generic configuration file reader that opens a file, reads the lines,
1224         handles continuation lines, comments, empty lines and feed all read lines
1225         into the function feeder(lineno, line).
1226         """
1227         
1228         f = open(cfgfile,'r')
1229         lineno = 0
1230         while 1:
1231                 lineno = lineno + 1
1232                 s = f.readline()
1233                 if not s: break
1234                 w = s.strip()
1235                 if not w: continue              # skip empty lines
1236                 s = s.rstrip()
1237                 if s[0] == '#': continue        # skip comments
1238                 while s[-1] == '\\':
1239                         s2 = f.readline()[:-1].strip()
1240                         s = s[:-1] + s2
1241                 feeder(lineno, s)
1242
1243
1244
1245
1246 # matches "VAR = VALUE"
1247 __config_regexp__  = re.compile( r"((?P<exp>export)\s*)*(?P<var>\w+)\s*(?P<colon>:)?=\s*(?P<apo>['\"]?)(?P<value>.*)(?P=apo)$")
1248
1249 # matches "include FILE"
1250 __include_regexp__ = re.compile( r"include\s+(.+)" )
1251
1252 __read_config_visit_cache__ = {}
1253
1254 def __read_config__(cfgfile, level):
1255         """Reads a configuration file"""
1256         visit = 1
1257
1258         def process_config(lineno, s):
1259                 s = s.strip()
1260                 m = __config_regexp__.match(s)
1261                 if m:
1262                         groupd = m.groupdict()
1263                         key = groupd["var"]
1264                         if groupd.has_key("exp") and groupd["exp"] is not None:
1265                                 if not envflags.has_key(key):
1266                                         envflags[key] = {}
1267                                 envflags[key]["export"] = 1
1268                         if groupd.has_key("colon") and groupd["colon"] is not None:
1269                                 setenv(key, groupd["value"])
1270                         else:
1271                                 env[key] = groupd["value"]
1272 #                       print key,groupd["value"]
1273                         return
1274
1275                 m = __include_regexp__.match(s)
1276                 if m:
1277                         if not visit: return
1278                         file = expand(m.group(1))
1279                         if file[0] != '/':
1280                                 path = getenv('OEPATH').split(':')
1281                                 path.append(os.path.dirname(os.path.abspath(oefile)))
1282                                 for s in path:
1283                                         if os.access(os.path.join(s,file), os.R_OK):
1284                                                 file = os.path.join(s,file)
1285                         if os.access(file, os.R_OK):
1286                                 if level==0:
1287                                         inherit_os_env(2)
1288                                 __read_config__(file, level+1)
1289                         else:
1290                                 debug(1, "%s:%d: could not import %s" % (cfgfile, lineno, s))
1291                         return
1292
1293                 print lineno, s
1294
1295         cfgfile = os.path.abspath(cfgfile)
1296         if __read_config_visit_cache__.has_key(cfgfile): visit = 0
1297         __read_config_visit_cache__[cfgfile] = 1
1298         debug(2, "read " + cfgfile)
1299         reader(cfgfile, process_config)
1300
1301 def read_config(cfgfile):
1302         __read_config__(cfgfile, 0)
1303
1304         for s in ['BUILD_ARCH','BUILD_OS', 'ARCH', 'OS', 'MACHINE','ARCH']:
1305                 if not env.has_key(s):
1306                         print "FATAL: %s" % envflags[s]['warn']
1307                         fatal('read ${OEDIR}/oe.conf to learn about local configuration')
1308
1309
1310 __func_start_regexp__    = re.compile( r"((?P<py>python)\s*)*(?P<func>\w+)\s*\(\s*\)\s*{$" )
1311 __include_regexp__       = re.compile( r"include\s+(.+)" )
1312 __inherit_regexp__       = re.compile( r"inherit\s+(.+)" )
1313 __export_func_regexp__   = re.compile( r"EXPORT_FUNCTIONS\s+(.+)" )
1314 __addtask_regexp__       = re.compile("addtask\s+(?P<func>\w+)\s*((before\s*(?P<before>((.*(?=after))|(.*))))|(after\s*(?P<after>((.*(?=before))|(.*)))))*")
1315 __addhandler_regexp__    = re.compile( r"addhandler\s+(.+)" )
1316
1317 __read_oe_infunc__ = ""
1318 __read_oe_body__   = []
1319 __read_oe_classname__ = "" # our python equiv to OECLASS
1320 __oepath_found_it__ = 0
1321
1322 def read_oe(oefile, inherit = False, classname = None):
1323         """Reads a build file"""
1324         """When inherit flag is set to False(default), EXPORT_FUNCTIONS is ignored."""
1325
1326         def process_oe(lineno, s):
1327                 global __read_oe_infunc__, __read_oe_body__, __read_oe_classname__, __oepath_found_it__, __addhandler_regexp__
1328                 if __read_oe_infunc__:
1329                         if s == '}':
1330                                 __read_oe_body__.append('')
1331                                 env[__read_oe_infunc__] = string.join(__read_oe_body__, '\n')
1332                                 __read_oe_infunc__ = None
1333                                 __read_oe_body__ = []
1334                         else:
1335                                 try:
1336                                         if envflags[__read_oe_infunc__]["python"] == 1:
1337                                                 s = re.sub(r"^\t", '', s)
1338                                 except KeyError:
1339                                         pass
1340                                 __read_oe_body__.append(s)
1341                         return
1342                         
1343                 m = __config_regexp__.match(s)
1344                 if m:
1345                         key = m.group("var")
1346                         var = m.group("value")
1347                         if var and (var[0]=='"' or var[0]=="'"):
1348                                 fatal("Mismatch in \" or ' characters for %s=" % key)
1349                         if m.group("colon"):
1350                                 setenv(key, var)
1351                         else:
1352                                 env[key] = var
1353                         #print "%s=%s" % (key,var)
1354                         return
1355
1356                 m = __func_start_regexp__.match(s)
1357                 if m:
1358                         __read_oe_infunc__ = m.group("func")
1359                         key = __read_oe_infunc__
1360                         if m.group("py") is not None:
1361                                 if not envflags.has_key(key):
1362                                         envflags[key] = {}
1363                                 envflags[key]["python"] = 1
1364                         return
1365
1366                 m = __include_regexp__.match(s)
1367                 if m:
1368                         file = expand(m.group(1))
1369                         if file[0] != '/':
1370                                 path = getenv('OEPATH').split(':')
1371                                 path.append(os.path.dirname(os.path.abspath(oefile)))
1372                                 for s in path:
1373                                         if os.access(os.path.join(s,file), os.R_OK):
1374                                                 file = os.path.join(s,file)
1375                         try:
1376                                 read_oe(file)
1377                         except IOError:
1378                                 fatal("error accessing build file %s" % file)
1379                         return
1380
1381                 m = __inherit_regexp__.match(s)
1382                 if m:
1383                         __word__ = re.compile(r"\S+")
1384                         files = m.group(1)
1385                         n = __word__.findall(files)
1386                         for f in n:
1387                                 file = expand(f)
1388                                 if file[0] != "/":
1389                                         if env.has_key('OEPATH'):
1390                                                 __oepath_found_it__ = 0
1391                                                 for dir in expand(env['OEPATH']).split(":"):
1392 #                                                       print "attempting to access %s" % os.path.join(dir, "classes",file + ".oeclass")
1393                                                         if os.access(os.path.join(dir, "classes", file + ".oeclass"), os.R_OK):
1394                                                                 file = os.path.join(dir, "classes",file + ".oeclass")
1395                                                                 __oepath_found_it__ = 1
1396                                         if __oepath_found_it__ == 0:
1397                                                 fatal("unable to locate %s in OEPATH"  % file)
1398                                         __read_oe_classname__ = file
1399
1400                                 o = re.match(r".*/([^/\.]+)",file)
1401                                 if o:
1402                                         __read_oe_classname__ = o.group(1)
1403
1404 #                               print "read_oe: inherit: loading %s" % file
1405                                 try:
1406                                         read_oe(file, True, __read_oe_classname__)
1407                                         __read_oe_classname__ = classname
1408                                 except IOError:
1409                                         fatal("error accessing build file %s" % file)
1410                         return
1411
1412                 __word__ = re.compile(r"\S+")
1413
1414                 m = __export_func_regexp__.match(s)
1415                 if m:
1416                         if inherit == True:
1417                                 fns = m.group(1)
1418                                 n = __word__.findall(fns)
1419                                 for f in n:
1420                                         
1421                                         oldfunc = "%s_%s" % (__read_oe_classname__, f)
1422                                         if envflags.has_key(oldfunc) and  envflags[oldfunc].has_key("python"):
1423                                                 setenv(f, "exec_task('%s','S')\n" % oldfunc)
1424                                                 if not envflags.has_key(f):
1425                                                         envflags[f] = {}
1426                                                 envflags[f]["python"] = 1
1427                                         else:
1428                                                 setenv(f, "\t%s\n" % oldfunc)
1429                         return
1430
1431
1432                 m = __addtask_regexp__.match(s)
1433                 if m:
1434                         func = m.group("func")
1435                         before = m.group("before")
1436                         after = m.group("after")
1437                         if func is None:
1438                                 return
1439                         var = "do_" + func
1440
1441                         if not envflags.has_key(var):
1442                                 envflags[var] = {}
1443
1444                         envflags[var]["task"] = "1"
1445
1446                         if after is not None:
1447                                 # set up deps for function
1448                                 envflags[var]["deps"] = after.split()
1449                         if before is not None:
1450                                 # set up things that depend on this func 
1451                                 envflags[var]["postdeps"] = before.split()
1452                         return
1453
1454                 m = __addhandler_regexp__.match(s)
1455                 if m:
1456                         fns = m.group(1)
1457                         hs = __word__.findall(fns)
1458                         for h in hs:
1459                                 if not envflags.has_key(h):
1460                                         envflags[h] = {}
1461                                 envflags[h]["handler"] = 1
1462                         return
1463
1464                 error("Unknown syntax in %s" % oefile)
1465                 print s
1466                 sys.exit(1)
1467
1468
1469         debug(2,"read_oe('%s')" % oefile)
1470         reader(oefile, process_oe)
1471
1472
1473
1474
1475 #######################################################################
1476 #######################################################################
1477 #
1478 # SECTION: Environment
1479 #
1480 # PURPOSE: store, modify and emit environment variables. Enforce need
1481 #          variables, calculate missing ones.
1482 #
1483 #######################################################################
1484 #######################################################################
1485
1486 __expand_var_regexp__ = re.compile(r"\${[^{}]+}")
1487 __expand_python_regexp__ = re.compile(r"\${@.+?}")
1488
1489 def expand(s, env = globals()["env"]):
1490         """Can expand variables with their values from env[]
1491
1492         >>> env['MID'] = 'drin'
1493         >>> print expand('vorher ${MID} dahinter')
1494         vorher drin dahinter
1495
1496         Unset variables are kept as is:
1497
1498         >>> print expand('vorher ${MID} dahinter ${UNKNOWN}')
1499         vorher drin dahinter ${UNKNOWN}
1500
1501         A syntax error just returns the string:
1502
1503         >>> print expand('${UNKNOWN')
1504         ${UNKNOWN
1505
1506         We can evaluate python code:
1507
1508         >>> print expand('${@ "Test"*3}')
1509         TestTestTest
1510         >>> env['START'] = '0x4000'
1511         >>> print expand('${@ hex(0x1000000+${START}) }')
1512         0x1004000
1513
1514         We are able to handle recursive definitions:
1515
1516         >>> env['ARCH'] = 'arm'
1517         >>> env['OS'] = 'linux'
1518         >>> env['SYS'] = '${ARCH}-${OS}'
1519         >>> print expand('${SYS}')
1520         arm-linux
1521         """
1522
1523         def var_sub(match):
1524                 key = match.group()[2:-1]
1525                 #print "got key:", key
1526                 if env.has_key(key):
1527                         return env[key]
1528                 else:
1529                         return match.group()
1530
1531         def python_sub(match):
1532                 code = match.group()[3:-1]
1533                 s = eval(code)
1534                 if type(s) == types.IntType: s = str(s)
1535                 return s
1536
1537         while s.find('$') != -1:
1538                 olds = s
1539                 s = __expand_var_regexp__.sub(var_sub, s)
1540                 s = __expand_python_regexp__.sub(python_sub, s)
1541                 if len(s)>2048:
1542                         fatal("expanded string too long")
1543                 if s == olds: break
1544         return s
1545
1546
1547 #######################################################################
1548
1549 def setenv(var, value, env = globals()["env"]):
1550         """Simple set an environment in the global oe.env[] variable, but
1551         with expanding variables beforehand.
1552         """
1553
1554         env[var] = expand(value)
1555
1556
1557 #######################################################################
1558
1559 def getenv(var, env = globals()["env"]):
1560         """Returns an expanded environment var"""
1561         return expand('${%s}' % var, env)
1562
1563
1564 #######################################################################
1565
1566 def inherit_os_env(position, env = globals()["env"]):
1567         """This reads the the os-environment and imports variables marked as
1568         such in envflags into our environment. This happens at various places
1569         during package definition read time, see comments near envflags[] for
1570         more.
1571         """
1572         
1573         debug(2,"inherit_os_env(%d)" % position)
1574
1575         position = str(position)
1576         for s in os.environ.keys():
1577                 try:
1578                         d = envflags[s]
1579                         if d.has_key('inherit') and d['inherit'] == position:
1580                                 env[s] = os.environ[s]
1581                                 debug(2, 'inherit %s from os environment' % s)
1582                 except KeyError:
1583                         pass
1584
1585
1586 #######################################################################
1587
1588 def set_automatic_vars(file):
1589         """Deduce per-package environment variables"""
1590
1591         debug(2,"setting automatic vars")
1592         pkg = catpkgsplit(file)
1593         if pkg == None:
1594                 fatal("package file not in valid format")
1595         env['CATEGORY'] =       pkg[0]
1596         env['PN'] =             pkg[1]
1597         env['PV'] =             pkg[2]
1598         env['PR'] =             pkg[3]
1599         env['P'] =              '${PN}-${PV}'
1600         env['PF'] =             '${P}-${PR}'
1601
1602         for s in ['${TOPDIR}/${CATEGORY}/${PF}', 
1603                   '${TOPDIR}/${CATEGORY}/${PN}-${PV}',
1604                   '${TOPDIR}/${CATEGORY}/files',
1605                   '${TOPDIR}/${CATEGORY}']:
1606                 s = expand(s)
1607                 if os.access(s, os.R_OK):
1608                         env['FILESDIR'] =  s
1609                         break
1610         env['WORKDIR'] =        '${TMPDIR}/${CATEGORY}/${PF}'
1611         env['T'] =              '${WORKDIR}/temp'
1612         env['D'] =              '${WORKDIR}/image'
1613         env['S'] =              '${WORKDIR}/${P}'
1614         env['SLOT'] =           '0'
1615         inherit_os_env(3)
1616
1617
1618 #######################################################################
1619
1620 def set_additional_vars():
1621         """Deduce rest of variables, e.g. ${A} out of ${SRC_URI}"""
1622
1623         debug(2,"set_additional_vars")
1624
1625         inherit_os_env(4)
1626         if env.has_key('SRC_URI'):
1627                 # Do we already have something in A?
1628                 if env.has_key('A'):
1629                         a = env['A'].split()
1630                 else:
1631                         a = []
1632
1633                 import oe.fetch
1634                 try:
1635                         oe.fetch.init(getenv('SRC_URI').split())
1636                 except oe.fetch.NoMethodError:
1637                         (type, value, traceback) = sys.exc_info()
1638                         fatal("No method: %s" % value)
1639                         return
1640
1641                 a += oe.fetch.localpaths()
1642                 del oe.fetch
1643                 env['A'] = string.join(a)
1644
1645         for s in ['S','STAGING_DIR','STAGING_BINLIB', 'STAGING_LIBDIR']:
1646                 if env.has_key(s):
1647                         env[s] = getenv(s)
1648
1649
1650 #######################################################################
1651
1652 def update_env(env = globals()["env"]):
1653         """Modifies the environment vars according to local overrides
1654
1655         For the example we do some preparations:
1656
1657         >>> setenv('TEST_arm', 'target')
1658         >>> setenv('TEST_ramses', 'machine')
1659         >>> setenv('TEST_local', 'local')
1660         >>> setenv('OVERRIDES', 'arm')
1661
1662         and then we set some TEST environment variable and let it update:
1663
1664         >>> setenv('TEST', 'original')
1665         >>> update_env()
1666         >>> print env['TEST']
1667         target
1668
1669         You can set OVERRIDES to another value, yielding another result:
1670
1671         >>> setenv('OVERRIDES', 'arm:ramses:local')
1672         >>> setenv('TEST', 'original')
1673         >>> update_env()
1674         >>> print env['TEST']
1675         local
1676
1677         Besides normal updates, we are able to append text:
1678
1679         >>> setenv('TEST_append', ' foo')
1680         >>> update_env()
1681         >>> print env['TEST']
1682         local foo
1683
1684         And we can prepend text:
1685
1686         >>> setenv('TEST_prepend', 'more ')
1687         >>> update_env()
1688         >>> print env['TEST']
1689         more local foo
1690
1691         Deleting stuff is more fun with multiline environment variables, but
1692         it works with normal ones, too. The TEST_delete='foo' construct
1693         deletes any line in TEST that matches 'foo':
1694
1695         >>> setenv('TEST_delete', 'foo ')
1696         >>> update_env()
1697         >>> print "'%s'" % env['TEST']
1698         ''
1699         """
1700
1701         debug(2, "update_env()")
1702
1703         # can't do delete env[...] while iterating over the dictionary, so remember them
1704         dodel = []
1705         # preprocess overrides
1706         override = expand(env['OVERRIDES']).split(':')
1707
1708         for s in env:
1709                 for o in override:
1710                         name = s + '_' + o
1711                         if env.has_key(name):
1712                                 env[s] = env[name]
1713                                 dodel.append(name)
1714
1715                 # Handle line appends:
1716                 name = s+'_append'
1717                 if env.has_key(name):
1718                         env[s] = env[s]+env[name]
1719                         dodel.append(name)
1720
1721                 # Handle line prepends
1722                 name = s+'_prepend'
1723                 if env.has_key(name):
1724                         env[s] = env[name]+env[s]
1725                         dodel.append(name)
1726
1727                 # Handle line deletions
1728                 name = s+'_delete'
1729                 if env.has_key(name):
1730                         new = ''
1731                         pattern = string.replace(env[name],"\n","").strip()
1732                         for line in string.split(env[s],"\n"):
1733                                 if line.find(pattern) == -1:
1734                                         new = new + '\n' + line
1735                         env[s] = new
1736                         dodel.append(name)
1737
1738         # delete all environment vars no longer needed
1739         for s in dodel:
1740                 del env[s]
1741
1742         inherit_os_env(5)
1743
1744
1745 #######################################################################
1746
1747 def emit_env(o=sys.__stdout__, env = globals()["env"]):
1748         """This prints the contents of env[] so that it can later be sourced by a shell
1749         Normally, it prints to stdout, but this it can be redirectory to some open file handle
1750
1751         It is used by exec_shell_func().
1752         """
1753
1754 #       o.write('\nPATH="' + os.path.join(projectdir, 'bin/build') + ':${PATH}"\n')
1755
1756         for s in env:
1757                 if s == s.lower(): continue
1758
1759                 o.write('\n')
1760                 if envflags.has_key(s):
1761                         if envflags[s].has_key('python') or envflags.has_key('handler'):
1762                                 continue
1763                         if envflags[s].has_key('export'):
1764                                  o.write('export ')
1765
1766                 # if we're going to output this within doublequotes,
1767                 # to a shell, we need to escape the quotes in the var
1768                 alter = re.sub('"', '\\"', getenv(s,env)) 
1769                 o.write(s+'="'+alter+'"\n')
1770
1771         for s in env:
1772                 if s != s.lower(): continue
1773                 if envflags.has_key(s):
1774                         if envflags[s].has_key('python') or envflags.has_key('handler'):
1775                                 continue
1776
1777                 # NOTE: should probably check for unbalanced {} within the var
1778                 o.write("\n" + s + '() {\n' + getenv(s,env) + '}\n')
1779
1780
1781 #######################################################################
1782
1783 def print_orphan_env():
1784         """Debug output: do we have any variables that are not mentioned in oe.envflags[] ?"""
1785         for s in env:
1786                 if s == s.lower(): continue             # only care for env vars
1787                 header = 0                              # header shown?
1788                 try:
1789                         d = envflags[s]
1790                 except KeyError:
1791                         if not header:
1792                                 note("Nonstandard variables defined in your project:")
1793                                 header = 1
1794                         print debug_prepend + s
1795                 if header:
1796                         print
1797
1798
1799 #######################################################################
1800
1801 def print_missing_env():
1802         """Debug output: warn about all missing variables
1803
1804         Returns 1 on error, terminates on fatal error.
1805         """
1806
1807         err = 0
1808         for s in envflags:
1809                 if not envflags[s].has_key('warnlevel'): continue
1810                 if env.has_key(s): continue
1811
1812                 level = envflags[s]['warnlevel']
1813                 try: warn = debug_prepend + envflags[s]['warn']
1814                 except KeyError: warn = ''
1815                 if level == 1:
1816                         note('Variable %s is not defined' % s)
1817                         if warn: print " -", warn
1818                 elif level == 2:
1819                         error('Important variable %s is not defined' % s)
1820                         err = 1
1821                         if warn: print " -", warn
1822                 elif level == 3:
1823                         error('Important variable %s is not defined' % s)
1824                         if warn: print " -", warn
1825                         sys.exit(1)
1826         return err
1827
1828
1829 envflags = {
1830
1831 #
1832 # desc:        descriptional text
1833 #
1834 # warnlevel 1: notify the user that this field is missing
1835 # warnlevel 2: complain loudly if this field doesn't exist, but continue
1836 # warnlevel 3: this field is absolutely mandatory, stop working if it isn't there
1837 #
1838 # warn:        text to display when showing a warning
1839 #
1840 # inherit 1:   get this var from the environment (if possible) before global config
1841 # inherit 2:   between global and local config
1842 # inherit 3:   between local config and package definition
1843 # inherit 4:   before setting additional vars
1844 # inherit 5:   after package definition
1845 #              (this defines the precendency, because any var can be overriden by the next step)
1846 #
1847 # export:      when creating the package build script, do not only define this var, but export it
1848 #
1849
1850
1851 # Directories for the Build system
1852
1853 "OEDIR":                { "warnlevel": 3,
1854                           "inherit": "1" },
1855 "OEPATH":               { "warnlevel": 3,
1856                           "inherit": "1" },
1857 "TOPDIR":               { "warnlevel": 3,
1858                           "desc":       "Toplevel directory of build area" },
1859 "TMPDIR":               { "warnlevel": 3 },
1860 "DL_DIR":               { "warnlevel": 3 },
1861 "STAMP":                { "warnlevel": 3 },
1862 "STAGING_DIR":          { "warnlevel": 3 },
1863 "STAGING_BINDIR":       { "warnlevel": 3 },
1864 "STAGING_LIBDIR":       { "warnlevel": 3 },
1865
1866 # Mirros and download:
1867
1868 "DEBIAN_MIRROR":        { "warnlevel": 3 },
1869 "SOURCEFORGE_MIRROR":   { "warnlevel": 3 },
1870 "FETCHCOMMAND":         { "warnlevel": 3 },
1871 "RESUMECOMMAND":        { "warnlevel": 3 },
1872
1873 # Architecture / Board related:
1874
1875 "DISTRO":               { "warnlevel": 0, },
1876 "BUILD_ARCH":           { "warnlevel": 3,
1877                           "warn":      "put something like BUILD_ARCH='i686' into ${OEDIR}/conf/local.conf" },
1878 "BUILD_OS":             { "warnlevel": 3,
1879                           "warn":      "put something like BUILD_OS='linux' into ${OEDIR}/conf/local.conf" },
1880 "ARCH":                 { "warnlevel": 3,
1881                           "warn":      "put something like ARCH='arm' into ${OEDIR}/conf/local.conf" },
1882 "OS":                   { "warnlevel": 3,
1883                           "warn":      "put something like OS='linux' into ${OEDIR}/conf/local.conf" },
1884 "MACHINE":              { "warnlevel": 3,
1885                           "warn":      "put something like MACHINE='ramses' into ${OEDIR}/conf/local.conf" },
1886 "USE":                  { "warnlevel": 2,
1887                           "warn":      "put USE= with a list of features into ${OEDIR}/conf/local.conf" },
1888 "SYS":                  { "warnlevel": 3 },
1889 "BUILD_SYS":            { "warnlevel": 3 },
1890 "CROSS":                { "warnlevel": 3 },
1891 "OVERRIDES":            { "warnlevel": 2 },
1892 "ALLOWED_FLAGS":        { "warnlevel": 2 },
1893
1894 "FULL_OPTIMIZATION":    { "warnlevel": 2 },
1895 "OPTIMIZATION":         { "warnlevel": 2 },
1896
1897 "CPPFLAGS":             { "warnlevel": 3 },
1898 "CFLAGS":               { "warnlevel": 3 },
1899 "CXXFLAGS":             { "warnlevel": 3 },
1900 "LDFLAGS":              { "warnlevel": 3 },
1901 "CPP":                  { "warnlevel": 3 },
1902 "CC":                   { "warnlevel": 3 },
1903 "CXX":                  { "warnlevel": 3 },
1904 "LD":                   { "warnlevel": 3 },
1905 "STRIP":                { "warnlevel": 3 },
1906 "AR":                   { "warnlevel": 3 },
1907 "RANLIB":               { "warnlevel": 3 },
1908 "MAKE":                 { "warnlevel": 3 },
1909
1910 "BUILD_CPPFLAGS":       { "warnlevel": 3 },
1911 "BUILD_CFLAGS":         { "warnlevel": 3 },
1912 "BUILD_CXXFLAGS":       { "warnlevel": 3 },
1913 "BUILD_LDFLAGS":        { "warnlevel": 3 },
1914 "BUILD_CPP":            { "warnlevel": 3 },
1915 "BUILD_CC":             { "warnlevel": 3 },
1916 "BUILD_CXX":            { "warnlevel": 3 },
1917 "BUILD_LD":             { "warnlevel": 3 },
1918
1919 "PKG_CONFIG_PATH":      { "warnlevel": 3 },
1920
1921
1922
1923 # Mandatory fields in build files
1924
1925 "DESCRIPTION":          { "warnlevel": 3 },
1926 "DEPEND":               { "warnlevel": 1 },
1927 "RDEPEND":              { "warnlevel": 1 },
1928 "PROVIDES":             { "warnlevel": 0 },
1929 "SRC_URI":              { "warnlevel": 1 },
1930 "LICENSE":              { "warnlevel": 1 },
1931 "HOMEPAGE":             { "warnlevel": 1 },
1932
1933 # Use when needed
1934
1935 "PROVIDE":              { "warnlevel": 0 },
1936 "RECOMMEND":            { "warnlevel": 0 },
1937 "FOR_TARGET":           { "warnlevel": 0 },
1938 "SLOT":                 { "warnlevel": 0 },
1939 "GET_URI":              { "warnlevel": 0 },
1940 "MAINTAINER":           { "warnlevel": 0 },
1941 "EXTRA_OECONF":         { "warnlevel": 0 },
1942 "EXTRA_OEMAKE":         { "warnlevel": 0 },
1943
1944
1945
1946 #"OEBUILD":             { "warnlevel": 3 },
1947 "P":                    { "warnlevel": 3 },
1948 "CATEGORY":             { "warnlevel": 2 },
1949 "PN":                   { "warnlevel": 3 },
1950 "PV":                   { "warnlevel": 3 },
1951 "PR":                   { "warnlevel": 2 },
1952 "PF":                   { "warnlevel": 3 },
1953 "WORKDIR":              { "warnlevel": 3 },
1954 "FILESDIR":             { "warnlevel": 3 },
1955 "S":                    { "warnlevel": 3 },
1956 "T":                    { "warnlevel": 3 },
1957 "D":                    { "warnlevel": 3 },
1958 "A":                    { "warnlevel": 1 },
1959
1960 # Package creation functions:
1961
1962 "do_unpack":            { "warnlevel": 1 },
1963 "do_compile":           { "warnlevel": 1 },
1964 "do_stage":             { "warnlevel": 1 },
1965 "do_install":           { "warnlevel": 1 },
1966 "pkg_preinst":          { "warnlevel": 0 },
1967 "pkg_postint":          { "warnlevel": 0 },
1968 "pkg_prerm":            { "warnlevel": 0 },
1969 "pkg_postrm":           { "warnlevel": 0 },
1970
1971 # Automatically generated, but overrideable:
1972
1973 "OEDEBUG":              { "inherit":  "1" },
1974
1975 }
1976
1977 # defaults for vars needed to access oe.conf
1978 env['TOPDIR'] = projectdir
1979 env['OEDIR'] = os.path.join(sys.prefix, "share/oe")
1980 env['OEPATH'] = "${OEDIR}/bin:${OEDIR}:${TOPDIR}/bin:${TOPDIR}"
1981 inherit_os_env(1)
1982
1983 if __name__ == "__main__":
1984         import doctest, oe
1985         doctest.testmod(oe)