Enhance cvs fetching step one - obey a cvs checkout command from FETCHCOMMAND variabl...
[bitbake.git] / bin / oe / fetch.py
1 #!/usr/bin/python
2 """
3 OpenEmbedded 'Fetch' implementations
4
5 Classes for obtaining upstream sources for the
6 OpenEmbedded (http://openembedded.org) build infrastructure.
7
8 NOTE that it requires Python 2.x due to its use of static methods.
9
10 Copyright: (c) 2003 Chris Larson
11
12 Based on functions from the base oe module, Copyright 2003 Holger Schurig
13 """
14
15 import os, re, string
16 import oe
17
18 class FetchError(Exception):
19         """Exception raised when a download fails"""
20
21 class NoMethodError(Exception):
22         """Exception raised when there is no method to obtain a supplied url or set of urls"""
23
24 class MissingParameterError(Exception):
25         """Exception raised when a fetch method is missing a critical parameter in the url"""
26
27 methods = []
28
29 def init(urls = [], d = oe.data.init()):
30         for m in methods:
31                 m.urls = []
32
33         for u in urls:
34                 for m in methods:
35                         m.data = d
36                         if m.supports(u):
37                                 m.urls.append(u)
38         
39 def go(d = oe.data.init()):
40         """Fetch all urls"""
41         for m in methods:
42                 if m.urls:
43                         m.go(d)
44
45 def localpaths():
46         """Return a list of the local filenames, assuming successful fetch"""
47         local = []
48         for m in methods:
49                 for u in m.urls:
50                         local.append(m.localpath(u))
51         return local
52
53 def localpath(url):
54         for m in methods:
55                 if m.supports(url):
56                         return m.localpath(url)
57         return url 
58
59 class Fetch(object):
60         """Base class for 'fetch'ing data"""
61         
62         def __init__(self, urls = []):
63                 self.urls = []
64                 for url in urls:
65                         if self.supports(oe.decodeurl(url)) is 1:
66                                 self.urls.append(url)
67
68         def supports(url):
69                 """Check to see if this fetch class supports a given url.
70                    Expects supplied url in list form, as outputted by oe.decodeurl().
71                 """
72                 return 0
73         supports = staticmethod(supports)
74
75         def localpath(url):
76                 """Return the local filename of a given url assuming a successful fetch.
77                 """
78                 return url
79         localpath = staticmethod(localpath)
80
81         def setUrls(self, urls):
82                 self.__urls = urls
83
84         def getUrls(self):
85                 return self.__urls
86
87         urls = property(getUrls, setUrls, None, "Urls property")
88
89         def setData(self, data):
90                 self.__data = data
91
92         def getData(self):
93                 return self.__data
94
95         data = property(getData, setData, None, "Data property")
96
97         def go(self, urls = []):
98                 """Fetch urls"""
99                 raise NoMethodError("Missing implementation for url")
100
101 class Wget(Fetch):
102         """Class to fetch urls via 'wget'"""
103         def supports(url):
104                 """Check to see if a given url can be fetched using wget.
105                    Expects supplied url in list form, as outputted by oe.decodeurl().
106                 """
107                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
108                 return type in ['http','https','ftp']
109         supports = staticmethod(supports)
110
111         def localpath(url):
112                 # strip off parameters
113                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
114                 if parm.has_key("localpath"):
115                         # if user overrides local path, use it.
116                         return parm["localpath"]
117                 url = oe.encodeurl([type, host, path, user, pswd, {}])
118                 return os.path.join(oe.getenv("DL_DIR"), os.path.basename(url))
119         localpath = staticmethod(localpath)
120
121         def go(self, d = oe.data.init(), urls = []):
122                 """Fetch urls"""
123                 if not urls:
124                         urls = self.urls
125
126                 from copy import deepcopy
127                 localdata = deepcopy(d)
128                 oe.data.setVar('OVERRIDES', "wget:%s" % oe.data.getVar('OVERRIDES', localdata), localdata)
129                 oe.data.update_data(localdata)
130
131                 for loc in urls:
132                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(loc))
133                         myfile = os.path.basename(path)
134                         dlfile = self.localpath(loc)
135                         dlfile = oe.data.expand(dlfile, localdata)
136                         md5file = "%s.md5" % dlfile
137
138                         if os.path.exists(md5file):
139                                 # complete, nothing to see here..
140                                 continue
141
142                         if os.path.exists(dlfile):
143                                 # file exists, but we didnt complete it.. trying again..
144                                 myfetch = oe.data.expand(oe.data.getVar("RESUMECOMMAND", localdata), localdata)
145                         else:
146                                 myfetch = oe.data.expand(oe.data.getVar("FETCHCOMMAND", localdata), localdata)
147
148                         oe.note("fetch " +loc)
149                         myfetch = myfetch.replace("${URI}",oe.encodeurl([type, host, path, user, pswd, {}]))
150                         myfetch = myfetch.replace("${FILE}",myfile)
151                         oe.debug(2,myfetch)
152                         myret = os.system(myfetch)
153                         if myret != 0:
154                                 raise FetchError(myfile)
155
156                         # supposedly complete.. write out md5sum
157                         if oe.which(oe.data.getVar('PATH', d, 1), 'md5sum'):
158                                 md5pipe = os.popen('md5sum %s' % dlfile)
159                                 md5 = md5pipe.readline().split()[0]
160                                 md5pipe.close()
161                                 md5out = file(md5file, 'w')
162                                 md5out.write(md5)
163                                 md5out.close()
164                         else:
165                                 md5out = file(md5file, 'w')
166                                 md5out.write("")
167                                 md5out.close()
168                 del localdata
169                                                 
170
171 methods.append(Wget())
172
173 class Cvs(Fetch):
174         """Class to fetch a module or modules from cvs repositories"""
175         checkoutopts = { "tag": "-r",
176                          "date": "-D" }
177
178         def supports(url):
179                 """Check to see if a given url can be fetched with cvs.
180                    Expects supplied url in list form, as outputted by oe.decodeurl().
181                 """
182                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
183                 return type in ['cvs', 'pserver']
184         supports = staticmethod(supports)
185
186         def localpath(url):
187                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
188                 if parm.has_key("localpath"):
189                         # if user overrides local path, use it.
190                         return parm["localpath"]
191
192                 if not parm.has_key("module"):
193                         return url
194                 else:
195                         return os.path.join(oe.getenv("DL_DIR"), parm["module"])
196         localpath = staticmethod(localpath)
197
198         def go(self, d = oe.data.init(), urls = []):
199                 """Fetch urls"""
200                 if not urls:
201                         urls = self.urls
202
203                 from copy import deepcopy
204                 localdata = deepcopy(d)
205                 oe.data.setVar('OVERRIDES', "cvs:%s" % oe.data.getVar('OVERRIDES', localdata), localdata)
206                 oe.data.update_data(localdata)
207
208                 for loc in urls:
209                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(loc))
210                         if not parm.has_key("module"):
211                                 raise MissingParameterError("cvs method needs a 'module' parameter")
212                         else:
213                                 module = parm["module"]
214
215                         dlfile = self.localpath(loc)
216                         # if local path contains the cvs
217                         # module, consider the dir above it to be the
218                         # download directory
219                         pos = dlfile.find(module)
220                         if pos:
221                                 dldir = dlfile[:pos]
222                         else:
223                                 dldir = os.path.dirname(dlfile)
224
225                         options = []
226
227                         for opt in self.checkoutopts:
228                                 if parm.has_key(opt):
229                                         options.append(self.checkoutopts[opt] + " " + parm[opt])
230
231                         if parm.has_key("method"):
232                                 method = parm["method"]
233                         else:
234                                 method = "pserver"
235
236                         olddir = os.path.abspath(os.getcwd())
237                         os.chdir(oe.data.expand(dldir, localdata))
238                         cvsroot = ":" + method + ":" + user
239                         if pswd:
240                                 cvsroot += ":" + pswd
241                         cvsroot += "@" + host + ":" + path
242
243                         oe.data.setVar('CVSROOT', cvsroot, localdata)
244                         oe.data.setVar('CVSCOOPTS', string.join(options), localdata)
245                         oe.data.setVar('CVSMODULE', module, localdata)
246                         oe.note("fetch " + loc)
247                         cvscmd = oe.data.getVar('FETCHCOMMAND', localdata, 1)
248                         oe.debug(1, "Running %s" % cvscmd)
249                         myret = os.system(cvscmd)
250                         os.chdir(olddir)
251                         if myret != 0:
252                                 raise FetchError(module)
253                 del localdata
254
255 methods.append(Cvs())
256
257 class Bk(Fetch):
258         def supports(url):
259                 """Check to see if a given url can be fetched via bitkeeper.
260                    Expects supplied url in list form, as outputted by oe.decodeurl().
261                 """
262                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
263                 return type in ['bk']
264         supports = staticmethod(supports)
265
266 methods.append(Bk())
267
268 class Local(Fetch):
269         def supports(url):
270                 """Check to see if a given url can be fetched in the local filesystem.
271                    Expects supplied url in list form, as outputted by oe.decodeurl().
272                 """
273                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
274                 return type in ['file','patch']
275         supports = staticmethod(supports)
276
277         def localpath(url):
278                 """Return the local filename of a given url assuming a successful fetch.
279                 """
280                 return url.split("://")[1]
281         localpath = staticmethod(localpath)
282
283         def go(self, urls = []):
284                 """Fetch urls (no-op for Local method)"""
285                 # no need to fetch local files, we'll deal with them in place.
286                 return 1
287
288 methods.append(Local())