BUGFIX: cvs checkouts with username but not password were broken.
[bitbake.git] / bin / oe / fetch.py
1 """
2 OpenEmbedded 'Fetch' implementations
3
4 Classes for obtaining upstream sources for the
5 OpenEmbedded (http://openembedded.org) build infrastructure.
6
7 NOTE that it requires Python 2.x due to its use of static methods.
8
9 Copyright: (c) 2003 Chris Larson
10
11 Based on functions from the base oe module, Copyright 2003 Holger Schurig
12 """
13
14 import os, re, string
15 import oe
16 import oe.data
17
18 class FetchError(Exception):
19         """Exception raised when a download fails"""
20
21 class NoMethodError(Exception):
22         """Exception raised when there is no method to obtain a supplied url or set of urls"""
23
24 class MissingParameterError(Exception):
25         """Exception raised when a fetch method is missing a critical parameter in the url"""
26
27 methods = []
28
29 def init(urls = []):
30         for m in methods:
31                 m.urls = []
32
33         for u in urls:
34                 for m in methods:
35                         if m.supports(u):
36                                 m.urls.append(u)
37         
38 def go():
39         """Fetch all urls"""
40         for m in methods:
41                 if m.urls:
42                         m.go()
43
44 def localpaths():
45         """Return a list of the local filenames, assuming successful fetch"""
46         local = []
47         for m in methods:
48                 for u in m.urls:
49                         local.append(m.localpath(u))
50         return local
51
52 def localpath(url):
53         for m in methods:
54                 if m.supports(url):
55                         return m.localpath(url)
56         return url 
57
58 class Fetch(object):
59         """Base class for 'fetch'ing data"""
60         
61         def __init__(self, urls = []):
62                 self.urls = []
63                 for url in urls:
64                         if self.supports(oe.decodeurl(url)) is 1:
65                                 self.urls.append(url)
66
67         def supports(url):
68                 """Check to see if this fetch class supports a given url.
69                    Expects supplied url in list form, as outputted by oe.decodeurl().
70                 """
71                 return 0
72         supports = staticmethod(supports)
73
74         def localpath(url):
75                 """Return the local filename of a given url assuming a successful fetch.
76                 """
77                 return url
78         localpath = staticmethod(localpath)
79
80         def setUrls(self, urls):
81                 self.__urls = urls
82
83         def getUrls(self):
84                 return self.__urls
85
86         urls = property(getUrls, setUrls, None, "Urls property")
87
88         def go(self, urls = []):
89                 """Fetch urls"""
90                 raise NoMethodError("Missing implementation for url")
91
92 class Wget(Fetch):
93         """Class to fetch urls via 'wget'"""
94         def supports(url):
95                 """Check to see if a given url can be fetched using wget.
96                    Expects supplied url in list form, as outputted by oe.decodeurl().
97                 """
98                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
99                 return type in ['http','https','ftp']
100         supports = staticmethod(supports)
101
102         def localpath(url):
103                 # strip off parameters
104                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
105                 if parm.has_key("localpath"):
106                         # if user overrides local path, use it.
107                         return parm["localpath"]
108                 url = oe.encodeurl([type, host, path, user, pswd, {}])
109                 return os.path.join(oe.getenv("DL_DIR"), os.path.basename(url))
110         localpath = staticmethod(localpath)
111
112         def go(self, urls = []):
113                 """Fetch urls"""
114                 if not urls:
115                         urls = self.urls
116
117                 for loc in urls:
118                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(loc))
119                         myfile = os.path.basename(path)
120                         dlfile = self.localpath(loc)
121
122                         if os.path.exists(dlfile):
123                                 # if the file exists, check md5
124                                 # if no md5 or mismatch, attempt resume.
125                                 # regardless of exit code, move on.
126                                 myfetch = oe.getenv("RESUMECOMMAND")
127                                 oe.note("fetch " +loc)
128                                 myfetch = myfetch.replace("${URI}",oe.encodeurl([type, host, path, user, pswd, {}]))
129                                 myfetch = myfetch.replace("${FILE}",myfile)
130                                 oe.debug(2,myfetch)
131                                 myret = os.system(myfetch)
132                                 continue
133                         else:
134                                 myfetch = oe.getenv("FETCHCOMMAND")
135                                 oe.note("fetch " +loc)
136                                 myfetch = myfetch.replace("${URI}",oe.encodeurl([type, host, path, user, pswd, {}]))
137                                 myfetch = myfetch.replace("${FILE}",myfile)
138                                 oe.debug(2,myfetch)
139                                 myret = os.system(myfetch)
140                                 if myret != 0:
141                                         raise FetchError(myfile)
142
143 methods.append(Wget())
144
145 class Cvs(Fetch):
146         """Class to fetch a module or modules from cvs repositories"""
147         checkoutopts = { "tag": "-r",
148                          "date": "-D" }
149
150         def supports(url):
151                 """Check to see if a given url can be fetched with cvs.
152                    Expects supplied url in list form, as outputted by oe.decodeurl().
153                 """
154                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
155                 return type in ['cvs', 'pserver']
156         supports = staticmethod(supports)
157
158         def localpath(url):
159                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
160                 if parm.has_key("localpath"):
161                         # if user overrides local path, use it.
162                         return parm["localpath"]
163
164                 if not parm.has_key("module"):
165                         return url
166                 else:
167                         return os.path.join(oe.getenv("DL_DIR"), parm["module"])
168         localpath = staticmethod(localpath)
169
170         def go(self, urls = []):
171                 """Fetch urls"""
172                 if not urls:
173                         urls = self.urls
174
175                 for loc in urls:
176                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(loc))
177                         if not parm.has_key("module"):
178                                 raise MissingParameterError("cvs method needs a 'module' parameter")
179                         else:
180                                 module = parm["module"]
181
182                         dlfile = self.localpath(loc)
183                         # if local path contains the cvs
184                         # module, consider the dir above it to be the
185                         # download directory
186                         pos = dlfile.find(module)
187                         if pos:
188                                 dldir = dlfile[:pos]
189                         else:
190                                 dldir = os.path.dirname(dlfile)
191
192                         options = []
193
194                         for opt in self.checkoutopts:
195                                 if parm.has_key(opt):
196                                         options.append(self.checkoutopts[opt] + " " + parm[opt])
197
198                         if parm.has_key("method"):
199                                 method = parm["method"]
200                         else:
201                                 method = "pserver"
202
203                         os.chdir(oe.expand(dldir))
204                         cvsroot = ":" + method + ":" + user
205                         if pswd:
206                                 cvsroot += ":" + pswd
207                         cvsroot += "@" + host + ":" + path
208
209 #                       if method == "pserver":
210 #                               # Login to the server
211 #                               cvscmd = "cvs -d" + cvsroot + " login"
212 #                               myret = os.system(cvscmd)
213 #                               if myret != 0:
214 #                                       raise FetchError(module)
215
216                         cvscmd = "cvs -d" + cvsroot
217                         cvscmd += " checkout " + string.join(options) + " " + module 
218                         oe.note("fetch " + loc)
219                         oe.debug(1, "Running %s" % cvscmd)
220                         myret = os.system(cvscmd)
221                         if myret != 0:
222                                 raise FetchError(module)
223
224 methods.append(Cvs())
225
226 class Bk(Fetch):
227         def supports(url):
228                 """Check to see if a given url can be fetched via bitkeeper.
229                    Expects supplied url in list form, as outputted by oe.decodeurl().
230                 """
231                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
232                 return type in ['bk']
233         supports = staticmethod(supports)
234
235 methods.append(Bk())
236
237 class Local(Fetch):
238         def supports(url):
239                 """Check to see if a given url can be fetched in the local filesystem.
240                    Expects supplied url in list form, as outputted by oe.decodeurl().
241                 """
242                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
243                 return type in ['file','patch']
244         supports = staticmethod(supports)
245
246         def localpath(url):
247                 """Return the local filename of a given url assuming a successful fetch.
248                 """
249                 return url.split("://")[1]
250         localpath = staticmethod(localpath)
251
252         def go(self, urls = []):
253                 """Fetch urls (no-op for Local method)"""
254                 # no need to fetch local files, we'll deal with them in place.
255                 return 1
256
257 methods.append(Local())