Replacing B.has_key(A) calls with A in B.
[bitbake.git] / bin / oe / __init__.py
1 #!/usr/bin/python
2 """
3 OpenEmbedded Build System Python Library
4
5 Copyright: (c) 2003 by Holger Schurig
6
7 Part of this code has been shamelessly stolen from Gentoo's portage.py.
8 This source had GPL-2 as license, so the same goes for this file.
9
10 Please visit http://www.openembedded.org/phpwiki/ for more info.
11
12 Try "pydoc ./oe.py" to get some nice output.
13 """
14
15 __version__ = "1.1"
16
17 __all__ = [
18
19         "debug",
20         "note",
21         "error",
22         "fatal",
23
24         "mkdirhier",
25         "movefile",
26
27         "tokenize",
28         "evaluate",
29         "flatten",
30         "relparse",
31         "ververify",
32         "isjustname",
33         "isspecific",
34         "pkgsplit",
35         "catpkgsplit",
36         "vercmp",
37         "pkgcmp",
38         "dep_parenreduce",
39         "dep_opconvert",
40         "digraph",
41
42 # fetch
43         "decodeurl",
44         "encodeurl",
45
46 # modules
47         "parse",
48         "data",
49         "event",
50         "build",
51         "fetch",
52  ]
53
54 import sys,os,string,types,re
55
56 #projectdir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
57 projectdir = os.getcwd()
58 env = {}
59
60 class VarExpandError(Exception):
61         pass
62
63 class MalformedUrl(Exception):
64         """Exception raised when encountering an invalid url"""
65
66
67 #######################################################################
68 #######################################################################
69 #
70 # SECTION: Debug
71 #
72 # PURPOSE: little functions to make yourself known
73 #
74 #######################################################################
75 #######################################################################
76
77 debug_prepend = ''
78
79
80 def debug(lvl, *args):
81         if 'OEDEBUG' in env and (env['OEDEBUG'] >= str(lvl)):
82                 print debug_prepend + 'DEBUG:', string.join(args, '')
83
84 def note(*args):
85         print debug_prepend + 'NOTE:', string.join(args, '')
86
87 def error(*args):
88         print debug_prepend + 'ERROR:', string.join(args, '')
89
90 def fatal(*args):
91         print debug_prepend + 'ERROR:', string.join(args, '')
92         sys.exit(1)
93
94
95 #######################################################################
96 #######################################################################
97 #
98 # SECTION: File
99 #
100 # PURPOSE: Basic file and directory tree related functions
101 #
102 #######################################################################
103 #######################################################################
104
105 def mkdirhier(dir):
106         """Create a directory like 'mkdir -p', but does not complain if
107         directory already exists like os.makedirs
108         """
109
110         debug(3, "mkdirhier(%s)" % dir)
111         try:
112                 os.makedirs(dir)
113                 debug(2, "created " + dir)
114         except OSError, e:
115                 if e.errno != 17: raise e
116
117
118 #######################################################################
119
120 import stat
121
122 def movefile(src,dest,newmtime=None,sstat=None):
123         """Moves a file from src to dest, preserving all permissions and
124         attributes; mtime will be preserved even when moving across
125         filesystems.  Returns true on success and false on failure. Move is
126         atomic.
127         """
128
129         #print "movefile("+src+","+dest+","+str(newmtime)+","+str(sstat)+")"
130         try:
131                 if not sstat:
132                         sstat=os.lstat(src)
133         except Exception, e:
134                 print "!!! Stating source file failed... movefile()"
135                 print "!!!",e
136                 return None
137
138         destexists=1
139         try:
140                 dstat=os.lstat(dest)
141         except:
142                 dstat=os.lstat(os.path.dirname(dest))
143                 destexists=0
144
145         if destexists:
146                 if stat.S_ISLNK(dstat[stat.ST_MODE]):
147                         try:
148                                 os.unlink(dest)
149                                 destexists=0
150                         except Exception, e:
151                                 pass
152
153         if stat.S_ISLNK(sstat[stat.ST_MODE]):
154                 try:
155                         target=os.readlink(src)
156                         if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
157                                 os.unlink(dest)
158                         os.symlink(target,dest)
159 #                       os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
160                         os.unlink(src)
161                         return os.lstat(dest)
162                 except Exception, e:
163                         print "!!! failed to properly create symlink:"
164                         print "!!!",dest,"->",target
165                         print "!!!",e
166                         return None
167
168         renamefailed=1
169         if sstat[stat.ST_DEV]==dstat[stat.ST_DEV]:
170                 try:
171                         ret=os.rename(src,dest)
172                         renamefailed=0
173                 except Exception, e:
174                         import errno
175                         if e[0]!=errno.EXDEV:
176                                 # Some random error.
177                                 print "!!! Failed to move",src,"to",dest
178                                 print "!!!",e
179                                 return None
180                         # Invalid cross-device-link 'bind' mounted or actually Cross-Device
181
182         if renamefailed:
183                 didcopy=0
184                 if stat.S_ISREG(sstat[stat.ST_MODE]):
185                         try: # For safety copy then move it over.
186                                 shutil.copyfile(src,dest+"#new")
187                                 os.rename(dest+"#new",dest)
188                                 didcopy=1
189                         except Exception, e:
190                                 print '!!! copy',src,'->',dest,'failed.'
191                                 print "!!!",e
192                                 return None
193                 else:
194                         #we don't yet handle special, so we need to fall back to /bin/mv
195                         a=getstatusoutput("/bin/mv -f "+"'"+src+"' '"+dest+"'")
196                         if a[0]!=0:
197                                 print "!!! Failed to move special file:"
198                                 print "!!! '"+src+"' to '"+dest+"'"
199                                 print "!!!",a
200                                 return None # failure
201                 try:
202                         if didcopy:
203                                 missingos.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
204                                 os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
205                                 os.unlink(src)
206                 except Exception, e:
207                         print "!!! Failed to chown/chmod/unlink in movefile()"
208                         print "!!!",dest
209                         print "!!!",e
210                         return None
211
212         if newmtime:
213                 os.utime(dest,(newmtime,newmtime))
214         else:
215                 os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
216                 newmtime=sstat[stat.ST_MTIME]
217         return newmtime
218
219
220
221 #######################################################################
222 #######################################################################
223 #
224 # SECTION: Download
225 #
226 # PURPOSE: Download via HTTP, FTP, CVS, BITKEEPER, handling of MD5-signatures
227 #          and mirrors
228 #
229 #######################################################################
230 #######################################################################
231
232 def decodeurl(url):
233         """Decodes an URL into the tokens (scheme, network location, path,
234         user, password, parameters). 
235
236         >>> decodeurl("http://www.google.com/index.html")
237         ('http', 'www.google.com', '/index.html', '', '', {})
238
239         CVS url with username, host and cvsroot. The cvs module to check out is in the
240         parameters:
241
242         >>> decodeurl("cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg")
243         ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'})
244
245         Dito, but this time the username has a password part. And we also request a special tag
246         to check out.
247
248         >>> decodeurl("cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;module=familiar/dist/ipkg;tag=V0-99-81")
249         ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'})
250         """
251
252         #debug(3, "decodeurl('%s')" % url)
253         m = re.compile('([^:]*):/*(.+@)?([^/]+)(/[^;]+);?(.*)').match(url)
254         if not m:
255                 raise MalformedUrl(url)
256
257         type = m.group(1)
258         host = m.group(3)
259         path = m.group(4)
260         user = m.group(2)
261         parm = m.group(5)
262         #print "type:", type
263         #print "host:", host
264         #print "path:", path
265         #print "parm:", parm
266         if user:
267                 m = re.compile('([^:]+)(:?(.*))@').match(user)
268                 if m:
269                         user = m.group(1)
270                         pswd = m.group(3)
271         else:
272                 user = ''
273                 pswd = ''
274         #print "user:", user
275         #print "pswd:", pswd
276         #print
277         p = {}
278         if parm:
279                 for s in parm.split(';'):
280                         s1,s2 = s.split('=')
281                         p[s1] = s2
282                         
283         return (type, host, path, user, pswd, p)
284                 
285 #######################################################################
286
287 def encodeurl(decoded):
288         """Encodes a URL from tokens (scheme, network location, path,
289         user, password, parameters). 
290
291         >>> encodeurl(['http', 'www.google.com', '/index.html', '', '', {}])
292
293         "http://www.google.com/index.html"
294
295         CVS with username, host and cvsroot. The cvs module to check out is in the
296         parameters:
297
298         >>> encodeurl(['cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}])
299
300         "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg"
301
302         Dito, but this time the username has a password part. And we also request a special tag
303         to check out.
304
305         >>> encodeurl(['cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'}])
306
307         "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;module=familiar/dist/ipkg;tag=V0-99-81"
308         """
309
310         (type, host, path, user, pswd, p) = decoded
311
312         if not type or not host or not path:
313                 fatal("invalid or missing parameters for url encoding")
314
315         url = '%s://' % type
316         if user:
317                 url += "%s" % user
318                 if pswd:
319                         url += ":%s" % pswd
320                 url += "@"
321         url += "%s%s" % (host, path)
322         if p:
323                 for parm in p.keys():
324                         url += ";%s=%s" % (parm, p[parm])
325
326         return url
327
328 #######################################################################
329
330 def which(path, item, direction = 1):
331         """Useful function for locating a file in a PATH"""
332         found = ""
333         for p in string.split(path or "", ":"):
334                 if os.path.exists(os.path.join(p, item)):
335                         found = os.path.join(p, item)
336                         if direction == 0:
337                                 break
338         return found
339
340 #######################################################################
341
342
343
344
345 #######################################################################
346 #######################################################################
347 #
348 # SECTION: Dependency
349 #
350 # PURPOSE: Compare build & run dependencies
351 #
352 #######################################################################
353 #######################################################################
354
355 def tokenize(mystring):
356         """Breaks a string like 'foo? (bar) oni? (blah (blah))' into (possibly embedded) lists:
357
358         >>> tokenize("x")
359         ['x']
360         >>> tokenize("x y")
361         ['x', 'y']
362         >>> tokenize("(x y)")
363         [['x', 'y']]
364         >>> tokenize("(x y) b c")
365         [['x', 'y'], 'b', 'c']
366         >>> tokenize("foo? (bar) oni? (blah (blah))")
367         ['foo?', ['bar'], 'oni?', ['blah', ['blah']]]
368         >>> tokenize("sys-apps/linux-headers nls? (sys-devel/gettext)")
369         ['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']]
370         """
371
372         newtokens = []
373         curlist   = newtokens
374         prevlists = []
375         level     = 0
376         accum     = ""
377         for x in mystring:
378                 if x=="(":
379                         if accum:
380                                 curlist.append(accum)
381                                 accum=""
382                         prevlists.append(curlist)
383                         curlist=[]
384                         level=level+1
385                 elif x==")":
386                         if accum:
387                                 curlist.append(accum)
388                                 accum=""
389                         if level==0:
390                                 print "!!! tokenizer: Unmatched left parenthesis in:\n'"+mystring+"'"
391                                 return None
392                         newlist=curlist
393                         curlist=prevlists.pop()
394                         curlist.append(newlist)
395                         level=level-1
396                 elif x in string.whitespace:
397                         if accum:
398                                 curlist.append(accum)
399                                 accum=""
400                 else:
401                         accum=accum+x
402         if accum:
403                 curlist.append(accum)
404         if (level!=0):
405                 print "!!! tokenizer: Exiting with unterminated parenthesis in:\n'"+mystring+"'"
406                 return None
407         return newtokens
408
409
410 #######################################################################
411
412 def evaluate(tokens,mydefines,allon=0):
413         """Removes tokens based on whether conditional definitions exist or not.
414         Recognizes !
415
416         >>> evaluate(['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']], {})
417         ['sys-apps/linux-headers']
418
419         Negate the flag:
420
421         >>> evaluate(['sys-apps/linux-headers', '!nls?', ['sys-devel/gettext']], {})
422         ['sys-apps/linux-headers', ['sys-devel/gettext']]
423
424         Define 'nls':
425
426         >>> evaluate(['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']], {"nls":1})
427         ['sys-apps/linux-headers', ['sys-devel/gettext']]
428
429         Turn allon on:
430
431         >>> evaluate(['sys-apps/linux-headers', 'nls?', ['sys-devel/gettext']], {}, True)
432         ['sys-apps/linux-headers', ['sys-devel/gettext']]
433         """
434
435         if tokens == None:
436                 return None
437         mytokens = tokens + []          # this copies the list
438         pos = 0
439         while pos < len(mytokens):
440                 if type(mytokens[pos]) == types.ListType:
441                         evaluate(mytokens[pos], mydefines)
442                         if not len(mytokens[pos]):
443                                 del mytokens[pos]
444                                 continue
445                 elif mytokens[pos][-1] == "?":
446                         cur = mytokens[pos][:-1]
447                         del mytokens[pos]
448                         if allon:
449                                 if cur[0] == "!":
450                                         del mytokens[pos]
451                         else:
452                                 if cur[0] == "!":
453                                         if (cur[1:] in mydefines) and (pos < len(mytokens)):
454                                                 del mytokens[pos]
455                                                 continue
456                                 elif (cur not in mydefines) and (pos < len(mytokens)):
457                                         del mytokens[pos]
458                                         continue
459                 pos = pos + 1
460         return mytokens
461
462
463 #######################################################################
464
465 def flatten(mytokens):
466         """Converts nested arrays into a flat arrays:
467
468         >>> flatten([1,[2,3]])
469         [1, 2, 3]
470         >>> flatten(['sys-apps/linux-headers', ['sys-devel/gettext']])
471         ['sys-apps/linux-headers', 'sys-devel/gettext']
472         """
473
474         newlist=[]
475         for x in mytokens:
476                 if type(x)==types.ListType:
477                         newlist.extend(flatten(x))
478                 else:
479                         newlist.append(x)
480         return newlist
481
482
483 #######################################################################
484
485 _package_weights_ = {"pre":-2,"p":0,"alpha":-4,"beta":-3,"rc":-1}       # dicts are unordered
486 _package_ends_    = ["pre", "p", "alpha", "beta", "rc", "cvs", "bk", "HEAD" ]                   # so we need ordered list
487
488 def relparse(myver):
489         """Parses the last elements of a version number into a triplet, that can
490         later be compared:
491
492         >>> relparse('1.2_pre3')
493         [1.2, -2, 3.0]
494         >>> relparse('1.2b')
495         [1.2, 98, 0]
496         >>> relparse('1.2')
497         [1.2, 0, 0]
498         """
499
500         number   = 0
501         p1       = 0
502         p2       = 0
503         mynewver = string.split(myver,"_")
504         if len(mynewver)==2:
505                 # an _package_weights_
506                 number = string.atof(mynewver[0])
507                 match = 0
508                 for x in _package_ends_:
509                         elen = len(x)
510                         if mynewver[1][:elen] == x:
511                                 match = 1
512                                 p1 = _package_weights_[x]
513                                 try:
514                                         p2 = string.atof(mynewver[1][elen:])
515                                 except:
516                                         p2 = 0
517                                 break
518                 if not match:   
519                         # normal number or number with letter at end
520                         divider = len(myver)-1
521                         if myver[divider:] not in "1234567890":
522                                 # letter at end
523                                 p1 = ord(myver[divider:])
524                                 number = string.atof(myver[0:divider])
525                         else:
526                                 number = string.atof(myver)             
527         else:
528                 # normal number or number with letter at end
529                 divider = len(myver)-1
530                 if myver[divider:] not in "1234567890":
531                         #letter at end
532                         p1     = ord(myver[divider:])
533                         number = string.atof(myver[0:divider])
534                 else:
535                         number = string.atof(myver)  
536         return [number,p1,p2]
537
538
539 #######################################################################
540
541 __ververify_cache__ = {}
542
543 def ververify(myorigval,silent=1):
544         """Returns 1 if given a valid version string, els 0. Valid versions are in the format
545
546         <v1>.<v2>...<vx>[a-z,_{_package_weights_}[vy]]
547
548         >>> ververify('2.4.20')
549         1
550         >>> ververify('2.4..20')                # two dots
551         0
552         >>> ververify('2.x.20')                 # 'x' is not numeric
553         0
554         >>> ververify('2.4.20a')
555         1
556         >>> ververify('2.4.20cvs')              # only one trailing letter
557         0
558         >>> ververify('1a')
559         1
560         >>> ververify('test_a')                 # no version at all
561         0
562         >>> ververify('2.4.20_beta1')
563         1
564         >>> ververify('2.4.20_beta')
565         1
566         >>> ververify('2.4.20_wrongext')        # _wrongext is no valid trailer
567         0
568         """
569
570         # Lookup the cache first
571         try:
572                 return __ververify_cache__[myorigval]
573         except KeyError:
574                 pass
575
576         if len(myorigval) == 0:
577                 if not silent:
578                         error("package version is empty")
579                 __ververify_cache__[myorigval] = 0
580                 return 0
581         myval = string.split(myorigval,'.')
582         if len(myval)==0:
583                 if not silent:
584                         error("package name has empty version string")
585                 __ververify_cache__[myorigval] = 0
586                 return 0
587         # all but the last version must be a numeric
588         for x in myval[:-1]:
589                 if not len(x):
590                         if not silent:
591                                 error("package version has two points in a row")
592                         __ververify_cache__[myorigval] = 0
593                         return 0
594                 try:
595                         foo = string.atoi(x)
596                 except:
597                         if not silent:
598                                 error("package version contains non-numeric '"+x+"'")
599                         __ververify_cache__[myorigval] = 0
600                         return 0
601         if not len(myval[-1]):
602                         if not silent:
603                                 error("package version has trailing dot")
604                         __ververify_cache__[myorigval] = 0
605                         return 0
606         try:
607                 foo = string.atoi(myval[-1])
608                 __ververify_cache__[myorigval] = 1
609                 return 1
610         except:
611                 pass
612
613         # ok, our last component is not a plain number or blank, let's continue
614         if myval[-1][-1] in string.lowercase:
615                 try:
616                         foo = string.atoi(myval[-1][:-1])
617                         return 1
618                         __ververify_cache__[myorigval] = 1
619                         # 1a, 2.0b, etc.
620                 except:
621                         pass
622         # ok, maybe we have a 1_alpha or 1_beta2; let's see
623         ep=string.split(myval[-1],"_")
624         if len(ep)!= 2:
625                 if not silent:
626                         error("package version has more than one letter at then end")
627                 __ververify_cache__[myorigval] = 0
628                 return 0
629         try:
630                 foo = string.atoi(ep[0])
631         except:
632                 # this needs to be numeric, i.e. the "1" in "1_alpha"
633                 if not silent:
634                         error("package version must have numeric part before the '_'")
635                 __ververify_cache__[myorigval] = 0
636                 return 0
637
638         for mye in _package_ends_:
639                 if ep[1][0:len(mye)] == mye:
640                         if len(mye) == len(ep[1]):
641                                 # no trailing numeric is ok
642                                 __ververify_cache__[myorigval] = 1
643                                 return 1
644                         else:
645                                 try:
646                                         foo = string.atoi(ep[1][len(mye):])
647                                         __ververify_cache__[myorigval] = 1
648                                         return 1
649                                 except:
650                                         # if no _package_weights_ work, *then* we return 0
651                                         pass    
652         if not silent:
653                 error("package version extension after '_' is invalid")
654         __ververify_cache__[myorigval] = 0
655         return 0
656
657
658 def isjustname(mypkg):
659         myparts = string.split(mypkg,'-')
660         for x in myparts:
661                 if ververify(x):
662                         return 0
663         return 1
664
665
666 _isspecific_cache_={}
667
668 def isspecific(mypkg):
669         "now supports packages with no category"
670         try:
671                 return __isspecific_cache__[mypkg]
672         except:
673                 pass
674
675         mysplit = string.split(mypkg,"/")
676         if not isjustname(mysplit[-1]):
677                         __isspecific_cache__[mypkg] = 1
678                         return 1
679         __isspecific_cache__[mypkg] = 0
680         return 0
681
682
683 #######################################################################
684
685 __pkgsplit_cache__={}
686
687 def pkgsplit(mypkg, silent=1):
688
689         """This function can be used as a package verification function. If
690         it is a valid name, pkgsplit will return a list containing:
691         [pkgname, pkgversion(norev), pkgrev ].
692
693         >>> pkgsplit('')
694         >>> pkgsplit('x')
695         >>> pkgsplit('x-')
696         >>> pkgsplit('-1')
697         >>> pkgsplit('glibc-1.2-8.9-r7')
698         >>> pkgsplit('glibc-2.2.5-r7')
699         ['glibc', '2.2.5', 'r7']
700         >>> pkgsplit('foo-1.2-1')
701         >>> pkgsplit('Mesa-3.0')
702         ['Mesa', '3.0', 'r0']
703         """
704
705         try:
706                 return __pkgsplit_cache__[mypkg]
707         except KeyError:
708                 pass
709
710         myparts = string.split(mypkg,'-')
711         if len(myparts) < 2:
712                 if not silent:
713                         error("package name without name or version part")
714                 __pkgsplit_cache__[mypkg] = None
715                 return None
716         for x in myparts:
717                 if len(x) == 0:
718                         if not silent:
719                                 error("package name with empty name or version part")
720                         __pkgsplit_cache__[mypkg] = None
721                         return None
722         # verify rev
723         revok = 0
724         myrev = myparts[-1]
725         ververify(myrev, 0)
726         if len(myrev) and myrev[0] == "r":
727                 try:
728                         string.atoi(myrev[1:])
729                         revok = 1
730                 except: 
731                         pass
732         if revok:
733                 if ververify(myparts[-2]):
734                         if len(myparts) == 2:
735                                 __pkgsplit_cache__[mypkg] = None
736                                 return None
737                         else:
738                                 for x in myparts[:-2]:
739                                         if ververify(x):
740                                                 __pkgsplit_cache__[mypkg]=None
741                                                 return None
742                                                 # names can't have versiony looking parts
743                                 myval=[string.join(myparts[:-2],"-"),myparts[-2],myparts[-1]]
744                                 __pkgsplit_cache__[mypkg]=myval
745                                 return myval
746                 else:
747                         __pkgsplit_cache__[mypkg] = None
748                         return None
749
750         elif ververify(myparts[-1],silent):
751                 if len(myparts)==1:
752                         if not silent:
753                                 print "!!! Name error in",mypkg+": missing name part."
754                         __pkgsplit_cache__[mypkg]=None
755                         return None
756                 else:
757                         for x in myparts[:-1]:
758                                 if ververify(x):
759                                         if not silent: error("package name has multiple version parts")
760                                         __pkgsplit_cache__[mypkg] = None
761                                         return None
762                         myval = [string.join(myparts[:-1],"-"), myparts[-1],"r0"]
763                         __pkgsplit_cache__[mypkg] = myval
764                         return myval
765         else:
766                 __pkgsplit_cache__[mypkg] = None
767                 return None
768
769
770 #######################################################################
771
772 __catpkgsplit_cache__ = {}
773
774 def catpkgsplit(mydata,silent=1):
775         """returns [cat, pkgname, version, rev ]
776
777         >>> catpkgsplit('sys-libs/glibc-1.2-r7')
778         ['sys-libs', 'glibc', '1.2', 'r7']
779         >>> catpkgsplit('glibc-1.2-r7')
780         ['null', 'glibc', '1.2', 'r7']
781         """
782
783         try:
784                 return __catpkgsplit_cache__[mydata]
785         except KeyError:
786                 pass
787
788         cat = os.path.basename(os.path.dirname(mydata))
789         mydata = os.path.join(cat, os.path.basename(mydata))
790 #       if mydata[:len(projectdir)] == projectdir:
791 #               mydata = mydata[len(projectdir)+1:]
792         if mydata[-3:] == '.oe':
793                 mydata = mydata[:-3]
794
795         mysplit = mydata.split("/")
796         p_split = None
797         splitlen = len(mysplit)
798         if splitlen == 1:
799                 retval = [None]
800                 p_split = pkgsplit(mydata,silent)
801         else:
802                 retval = [mysplit[splitlen - 2]]
803                 p_split = pkgsplit(mysplit[splitlen - 1],silent)
804         if not p_split:
805                 __catpkgsplit_cache__[mydata] = None
806                 return None
807         retval.extend(p_split)
808         __catpkgsplit_cache__[mydata] = retval
809         return retval
810
811
812 #######################################################################
813
814 __vercmp_cache__ = {}
815
816 def vercmp(val1,val2):
817         """This takes two version strings and returns an integer to tell you whether
818         the versions are the same, val1>val2 or val2>val1.
819         
820         >>> vercmp('1', '2')
821         -1.0
822         >>> vercmp('2', '1')
823         1.0
824         >>> vercmp('1', '1.0')
825         0
826         >>> vercmp('1', '1.1')
827         -1.0
828         >>> vercmp('1.1', '1_p2')
829         1.0
830         """
831
832         # quick short-circuit
833         if val1 == val2:
834                 return 0
835         valkey = val1+" "+val2
836
837         # cache lookup
838         try:
839                 return __vercmp_cache__[valkey]
840                 try:
841                         return - __vercmp_cache__[val2+" "+val1]
842                 except KeyError:
843                         pass
844         except KeyError:
845                 pass
846         
847         # consider 1_p2 vc 1.1
848         # after expansion will become (1_p2,0) vc (1,1)
849         # then 1_p2 is compared with 1 before 0 is compared with 1
850         # to solve the bug we need to convert it to (1,0_p2)
851         # by splitting _prepart part and adding it back _after_expansion
852
853         val1_prepart = val2_prepart = ''
854         if val1.count('_'):
855                 val1, val1_prepart = val1.split('_', 1)
856         if val2.count('_'):
857                 val2, val2_prepart = val2.split('_', 1)
858
859         # replace '-' by '.'
860         # FIXME: Is it needed? can val1/2 contain '-'?
861
862         val1 = string.split(val1,'-')
863         if len(val1) == 2:
864                 val1[0] = val1[0] +"."+ val1[1]
865         val2 = string.split(val2,'-')
866         if len(val2) == 2:
867                 val2[0] = val2[0] +"."+ val2[1]
868
869         val1 = string.split(val1[0],'.')
870         val2 = string.split(val2[0],'.')
871
872         # add back decimal point so that .03 does not become "3" !
873         for x in range(1,len(val1)):
874                 if val1[x][0] == '0' :
875                         val1[x] = '.' + val1[x]
876         for x in range(1,len(val2)):
877                 if val2[x][0] == '0' :
878                         val2[x] = '.' + val2[x]
879
880         # extend varion numbers
881         if len(val2) < len(val1):
882                 val2.extend(["0"]*(len(val1)-len(val2)))
883         elif len(val1) < len(val2):
884                 val1.extend(["0"]*(len(val2)-len(val1)))
885
886         # add back _prepart tails
887         if val1_prepart:
888                 val1[-1] += '_' + val1_prepart
889         if val2_prepart:
890                 val2[-1] += '_' + val2_prepart
891         # The above code will extend version numbers out so they
892         # have the same number of digits.
893         for x in range(0,len(val1)):
894                 cmp1 = relparse(val1[x])
895                 cmp2 = relparse(val2[x])
896                 for y in range(0,3):
897                         myret = cmp1[y] - cmp2[y]
898                         if myret != 0:
899                                 __vercmp_cache__[valkey] = myret
900                                 return myret
901         __vercmp_cache__[valkey] = 0
902         return 0
903
904
905 #######################################################################
906
907 def pkgcmp(pkg1,pkg2):
908         """ Compares two packages, which should have been split via
909         pkgsplit(). if the return value val is less than zero, then pkg2 is
910         newer than pkg1, zero if equal and positive if older.
911
912         >>> pkgcmp(['glibc', '2.2.5', 'r7'], ['glibc', '2.2.5', 'r7'])
913         0
914         >>> pkgcmp(['glibc', '2.2.5', 'r4'], ['glibc', '2.2.5', 'r7'])
915         -1
916         >>> pkgcmp(['glibc', '2.2.5', 'r7'], ['glibc', '2.2.5', 'r2'])
917         1
918         """
919         
920         mycmp = vercmp(pkg1[1],pkg2[1])
921         if mycmp > 0:
922                 return 1
923         if mycmp < 0:
924                 return -1
925         r1=string.atoi(pkg1[2][1:])
926         r2=string.atoi(pkg2[2][1:])
927         if r1 > r2:
928                 return 1
929         if r2 > r1:
930                 return -1
931         return 0
932
933
934 #######################################################################
935
936 def dep_parenreduce(mysplit, mypos=0):
937         """Accepts a list of strings, and converts '(' and ')' surrounded items to sub-lists:
938
939         >>> dep_parenreduce([''])
940         ['']
941         >>> dep_parenreduce(['1', '2', '3'])
942         ['1', '2', '3']
943         >>> dep_parenreduce(['1', '(', '2', '3', ')', '4'])
944         ['1', ['2', '3'], '4']
945         """
946
947         while mypos < len(mysplit): 
948                 if mysplit[mypos] == "(":
949                         firstpos = mypos
950                         mypos = mypos + 1
951                         while mypos < len(mysplit):
952                                 if mysplit[mypos] == ")":
953                                         mysplit[firstpos:mypos+1] = [mysplit[firstpos+1:mypos]]
954                                         mypos = firstpos
955                                         break
956                                 elif mysplit[mypos] == "(":
957                                         # recurse
958                                         mysplit = dep_parenreduce(mysplit,mypos)
959                                 mypos = mypos + 1
960                 mypos = mypos + 1
961         return mysplit
962
963
964 def dep_opconvert(mysplit, myuse):
965         "Does dependency operator conversion"
966         
967         mypos   = 0
968         newsplit = []
969         while mypos < len(mysplit):
970                 if type(mysplit[mypos]) == types.ListType:
971                         newsplit.append(dep_opconvert(mysplit[mypos],myuse))
972                         mypos += 1
973                 elif mysplit[mypos] == ")":
974                         # mismatched paren, error
975                         return None
976                 elif mysplit[mypos]=="||":
977                         if ((mypos+1)>=len(mysplit)) or (type(mysplit[mypos+1])!=types.ListType):
978                                 # || must be followed by paren'd list
979                                 return None
980                         try:
981                                 mynew = dep_opconvert(mysplit[mypos+1],myuse)
982                         except Exception, e:
983                                 error("unable to satisfy OR dependancy: " + string.join(mysplit," || "))
984                                 raise e
985                         mynew[0:0] = ["||"]
986                         newsplit.append(mynew)
987                         mypos += 2
988                 elif mysplit[mypos][-1] == "?":
989                         # use clause, i.e "gnome? ( foo bar )"
990                         # this is a quick and dirty hack so that repoman can enable all USE vars:
991                         if (len(myuse) == 1) and (myuse[0] == "*"):
992                                 # enable it even if it's ! (for repoman) but kill it if it's
993                                 # an arch variable that isn't for this arch. XXX Sparc64?
994                                 if (mysplit[mypos][:-1] not in settings.usemask) or \
995                                                 (mysplit[mypos][:-1]==settings["ARCH"]):
996                                         enabled=1
997                                 else:
998                                         enabled=0
999                         else:
1000                                 if mysplit[mypos][0] == "!":
1001                                         myusevar = mysplit[mypos][1:-1]
1002                                         enabled = not myusevar in myuse
1003                                         #if myusevar in myuse:
1004                                         #       enabled = 0
1005                                         #else:
1006                                         #       enabled = 1
1007                                 else:
1008                                         myusevar=mysplit[mypos][:-1]
1009                                         enabled = myusevar in myuse
1010                                         #if myusevar in myuse:
1011                                         #       enabled=1
1012                                         #else:
1013                                         #       enabled=0
1014                         if (mypos +2 < len(mysplit)) and (mysplit[mypos+2] == ":"):
1015                                 # colon mode
1016                                 if enabled:
1017                                         # choose the first option
1018                                         if type(mysplit[mypos+1]) == types.ListType:
1019                                                 newsplit.append(dep_opconvert(mysplit[mypos+1],myuse))
1020                                         else:
1021                                                 newsplit.append(mysplit[mypos+1])
1022                                 else:
1023                                         # choose the alternate option
1024                                         if type(mysplit[mypos+1]) == types.ListType:
1025                                                 newsplit.append(dep_opconvert(mysplit[mypos+3],myuse))
1026                                         else:
1027                                                 newsplit.append(mysplit[mypos+3])
1028                                 mypos += 4
1029                         else:
1030                                 # normal use mode
1031                                 if enabled:
1032                                         if type(mysplit[mypos+1]) == types.ListType:
1033                                                 newsplit.append(dep_opconvert(mysplit[mypos+1],myuse))
1034                                         else:
1035                                                 newsplit.append(mysplit[mypos+1])
1036                                 # otherwise, continue
1037                                 mypos += 2
1038                 else:
1039                         # normal item
1040                         newsplit.append(mysplit[mypos])
1041                         mypos += 1
1042         return newsplit
1043
1044 class digraph:
1045         """beautiful directed graph object"""
1046
1047         def __init__(self):
1048                 self.dict={}
1049                 #okeys = keys, in order they were added (to optimize firstzero() ordering)
1050                 self.okeys=[]
1051
1052         def __str__(self):
1053                 str = ""
1054                 for key in self.okeys:
1055                         str += "%s:\t%s\n" % (key, self.dict[key][1])
1056                 return str
1057
1058         def addnode(self,mykey,myparent):
1059                 if not mykey in self.dict:
1060                         self.okeys.append(mykey)
1061                         if myparent==None:
1062                                 self.dict[mykey]=[0,[]]
1063                         else:
1064                                 self.dict[mykey]=[0,[myparent]]
1065                                 self.dict[myparent][0]=self.dict[myparent][0]+1
1066                         return
1067                 if myparent and (not myparent in self.dict[mykey][1]):
1068                         self.dict[mykey][1].append(myparent)
1069                         self.dict[myparent][0]=self.dict[myparent][0]+1
1070         
1071         def delnode(self,mykey, ref = 1):
1072                 """Delete a node
1073
1074                 If ref is 1, remove references to this node from other nodes.
1075                 If ref is 2, remove nodes that reference this node."""
1076                 if not mykey in self.dict:
1077                         return
1078                 for x in self.dict[mykey][1]:
1079                         self.dict[x][0]=self.dict[x][0]-1
1080                 del self.dict[mykey]
1081                 while 1:
1082                         try:
1083                                 self.okeys.remove(mykey)        
1084                         except ValueError:
1085                                 break
1086                 if ref:
1087                         __kill = []
1088                         for k in self.okeys:
1089                                 if mykey in self.dict[k][1]:
1090                                         if ref == 1 or ref == 2:
1091                                                 self.dict[k][1].remove(mykey)
1092                                         if ref == 2:
1093                                                 __kill.append(k)
1094                         for l in __kill:
1095                                 self.delnode(l, ref)
1096         
1097         def allnodes(self):
1098                 "returns all nodes in the dictionary"
1099                 return self.dict.keys()
1100         
1101         def firstzero(self):
1102                 "returns first node with zero references, or NULL if no such node exists"
1103                 for x in self.okeys:
1104                         if self.dict[x][0]==0:
1105                                 return x
1106                 return None 
1107
1108         def firstnonzero(self):
1109                 "returns first node with nonzero references, or NULL if no such node exists"
1110                 for x in self.okeys:
1111                         if self.dict[x][0]!=0:
1112                                 return x
1113                 return None 
1114
1115
1116         def allzeros(self):
1117                 "returns all nodes with zero references, or NULL if no such node exists"
1118                 zerolist = []
1119                 for x in self.dict.keys():
1120                         if self.dict[x][0]==0:
1121                                 zerolist.append(x)
1122                 return zerolist
1123
1124         def hasallzeros(self):
1125                 "returns 0/1, Are all nodes zeros? 1 : 0"
1126                 zerolist = []
1127                 for x in self.dict.keys():
1128                         if self.dict[x][0]!=0:
1129                                 return 0
1130                 return 1
1131
1132         def empty(self):
1133                 if len(self.dict)==0:
1134                         return 1
1135                 return 0
1136
1137         def hasnode(self,mynode):
1138                 return mynode in self.dict
1139
1140         def getparents(self, item):
1141                 if not self.hasnode(item):
1142                         return []
1143                 return self.dict[item][1]
1144         
1145         def getchildren(self, item):
1146                 if not self.hasnode(item):
1147                         return []
1148                 children = [i for i in self.okeys if item in self.getparents(i)]
1149                 return children
1150         
1151         def walkdown(self, item, callback, debug = None):
1152                 __down_callback_cache = []
1153                 __recurse_count = 0
1154                 if not self.hasnode(item):
1155                         return 0
1156         
1157                 if __down_callback_cache.count(item):
1158                         return 1
1159
1160                 parents = self.getparents(item)
1161                 children = self.getchildren(item)
1162                 for p in parents:
1163                         if p in children:
1164 #                               print "%s is both parent and child of %s" % (p, item)
1165                                 __down_callback_cache.append(p)
1166                                 ret = callback(self, p)
1167                                 if ret == 0:
1168                                         return 0
1169                                 continue
1170                         if item == p:
1171                                 print "eek, i'm my own parent!"
1172                                 return 0
1173                         if debug:
1174                                 print "item: %s, p: %s" % (item, p)
1175                         ret = self.walkdown(p, callback, debug)
1176                         if ret == 0:
1177                                 return 0
1178
1179                 __down_callback_cache.append(item)
1180                 return callback(self, item)
1181         
1182         def walkup(self, item, callback):
1183                 if not self.hasnode(item):
1184                         return 0
1185         
1186                 parents = self.getparents(item)
1187                 children = self.getchildren(item)
1188                 for c in children:
1189                         if c in parents:
1190                                 ret = callback(self, item)
1191                                 if ret == 0:
1192                                         return 0
1193                                 continue
1194                         if item == c:
1195                                 print "eek, i'm my own child!"
1196                                 return 0
1197                         ret = self.walkup(c, callback)
1198                         if ret == 0:
1199                                 return 0
1200                 return callback(self, item)
1201
1202         def copy(self):
1203                 mygraph=digraph()
1204                 for x in self.dict.keys():
1205                         mygraph.dict[x]=self.dict[x][:]
1206                         mygraph.okeys=self.okeys[:]
1207                 return mygraph
1208
1209 #######################################################################
1210 #######################################################################
1211 #
1212 # SECTION: Config
1213 #
1214 # PURPOSE: Reading and handling of system/target-specific/local configuration
1215 #          reading of package configuration
1216 #
1217 #######################################################################
1218 #######################################################################
1219
1220 def reader(cfgfile, feeder):
1221         """Generic configuration file reader that opens a file, reads the lines,
1222         handles continuation lines, comments, empty lines and feed all read lines
1223         into the function feeder(lineno, line).
1224         """
1225         
1226         f = open(cfgfile,'r')
1227         lineno = 0
1228         while 1:
1229                 lineno = lineno + 1
1230                 s = f.readline()
1231                 if not s: break
1232                 w = s.strip()
1233                 if not w: continue              # skip empty lines
1234                 s = s.rstrip()
1235                 if s[0] == '#': continue        # skip comments
1236                 while s[-1] == '\\':
1237                         s2 = f.readline()[:-1].strip()
1238                         s = s[:-1] + s2
1239                 feeder(lineno, s)
1240
1241 if __name__ == "__main__":
1242         import doctest, oe
1243         doctest.testmod(oe)