Write out a .md5 file for the Wget downloads. Currently only being utilized as a...
[bitbake.git] / bin / oe / fetch.py
1 #!/usr/bin/python
2 """
3 OpenEmbedded 'Fetch' implementations
4
5 Classes for obtaining upstream sources for the
6 OpenEmbedded (http://openembedded.org) build infrastructure.
7
8 NOTE that it requires Python 2.x due to its use of static methods.
9
10 Copyright: (c) 2003 Chris Larson
11
12 Based on functions from the base oe module, Copyright 2003 Holger Schurig
13 """
14
15 import os, re, string
16 import oe
17
18 class FetchError(Exception):
19         """Exception raised when a download fails"""
20
21 class NoMethodError(Exception):
22         """Exception raised when there is no method to obtain a supplied url or set of urls"""
23
24 class MissingParameterError(Exception):
25         """Exception raised when a fetch method is missing a critical parameter in the url"""
26
27 methods = []
28
29 def init(urls = []):
30         for m in methods:
31                 m.urls = []
32
33         for u in urls:
34                 for m in methods:
35                         if m.supports(u):
36                                 m.urls.append(u)
37         
38 def go(d = oe.data.init()):
39         """Fetch all urls"""
40         for m in methods:
41                 if m.urls:
42                         m.go(d)
43
44 def localpaths():
45         """Return a list of the local filenames, assuming successful fetch"""
46         local = []
47         for m in methods:
48                 for u in m.urls:
49                         local.append(m.localpath(u))
50         return local
51
52 def localpath(url):
53         for m in methods:
54                 if m.supports(url):
55                         return m.localpath(url)
56         return url 
57
58 class Fetch(object):
59         """Base class for 'fetch'ing data"""
60         
61         def __init__(self, urls = []):
62                 self.urls = []
63                 for url in urls:
64                         if self.supports(oe.decodeurl(url)) is 1:
65                                 self.urls.append(url)
66
67         def supports(url):
68                 """Check to see if this fetch class supports a given url.
69                    Expects supplied url in list form, as outputted by oe.decodeurl().
70                 """
71                 return 0
72         supports = staticmethod(supports)
73
74         def localpath(url):
75                 """Return the local filename of a given url assuming a successful fetch.
76                 """
77                 return url
78         localpath = staticmethod(localpath)
79
80         def setUrls(self, urls):
81                 self.__urls = urls
82
83         def getUrls(self):
84                 return self.__urls
85
86         urls = property(getUrls, setUrls, None, "Urls property")
87
88         def go(self, urls = []):
89                 """Fetch urls"""
90                 raise NoMethodError("Missing implementation for url")
91
92 class Wget(Fetch):
93         """Class to fetch urls via 'wget'"""
94         def supports(url):
95                 """Check to see if a given url can be fetched using wget.
96                    Expects supplied url in list form, as outputted by oe.decodeurl().
97                 """
98                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
99                 return type in ['http','https','ftp']
100         supports = staticmethod(supports)
101
102         def localpath(url):
103                 # strip off parameters
104                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
105                 if parm.has_key("localpath"):
106                         # if user overrides local path, use it.
107                         return parm["localpath"]
108                 url = oe.encodeurl([type, host, path, user, pswd, {}])
109                 return os.path.join(oe.getenv("DL_DIR"), os.path.basename(url))
110         localpath = staticmethod(localpath)
111
112         def go(self, d = oe.data.init(), urls = []):
113                 """Fetch urls"""
114                 if not urls:
115                         urls = self.urls
116
117                 for loc in urls:
118                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(loc))
119                         myfile = os.path.basename(path)
120                         dlfile = self.localpath(loc)
121                         dlfile = oe.data.expand(dlfile, d)
122                         md5file = "%s.md5" % dlfile
123
124                         if os.path.exists(md5file):
125                                 # complete, nothing to see here..
126                                 continue
127
128                         if os.path.exists(dlfile):
129                                 # file exists, but we didnt complete it.. trying again..
130                                 myfetch = oe.data.expand(oe.data.getVar("RESUMECOMMAND", d), d)
131                         else:
132                                 myfetch = oe.data.expand(oe.data.getVar("FETCHCOMMAND", d), d)
133
134                         oe.note("fetch " +loc)
135                         myfetch = myfetch.replace("${URI}",oe.encodeurl([type, host, path, user, pswd, {}]))
136                         myfetch = myfetch.replace("${FILE}",myfile)
137                         oe.debug(2,myfetch)
138                         myret = os.system(myfetch)
139                         if myret != 0:
140                                 raise FetchError(myfile)
141
142                         # supposedly complete.. write out md5sum
143                         if oe.which(oe.data.getVar('PATH', d, 1), 'md5sum'):
144                                 md5pipe = os.popen('md5sum %s' % dlfile)
145                                 md5 = md5pipe.readline().split()[0]
146                                 md5pipe.close()
147                                 md5out = file(md5file, 'w')
148                                 md5out.write(md5)
149                                 md5out.close()
150                         else:
151                                 md5out = file(md5file, 'w')
152                                 md5out.write("")
153                                 md5out.close()
154                                                 
155                                                 
156
157 methods.append(Wget())
158
159 class Cvs(Fetch):
160         """Class to fetch a module or modules from cvs repositories"""
161         checkoutopts = { "tag": "-r",
162                          "date": "-D" }
163
164         def supports(url):
165                 """Check to see if a given url can be fetched with cvs.
166                    Expects supplied url in list form, as outputted by oe.decodeurl().
167                 """
168                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
169                 return type in ['cvs', 'pserver']
170         supports = staticmethod(supports)
171
172         def localpath(url):
173                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
174                 if parm.has_key("localpath"):
175                         # if user overrides local path, use it.
176                         return parm["localpath"]
177
178                 if not parm.has_key("module"):
179                         return url
180                 else:
181                         return os.path.join(oe.getenv("DL_DIR"), parm["module"])
182         localpath = staticmethod(localpath)
183
184         def go(self, d = oe.data.init(), urls = []):
185                 """Fetch urls"""
186                 if not urls:
187                         urls = self.urls
188
189                 for loc in urls:
190                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(loc))
191                         if not parm.has_key("module"):
192                                 raise MissingParameterError("cvs method needs a 'module' parameter")
193                         else:
194                                 module = parm["module"]
195
196                         dlfile = self.localpath(loc)
197                         # if local path contains the cvs
198                         # module, consider the dir above it to be the
199                         # download directory
200                         pos = dlfile.find(module)
201                         if pos:
202                                 dldir = dlfile[:pos]
203                         else:
204                                 dldir = os.path.dirname(dlfile)
205
206                         options = []
207
208                         for opt in self.checkoutopts:
209                                 if parm.has_key(opt):
210                                         options.append(self.checkoutopts[opt] + " " + parm[opt])
211
212                         if parm.has_key("method"):
213                                 method = parm["method"]
214                         else:
215                                 method = "pserver"
216
217                         olddir = os.path.abspath(os.getcwd())
218                         os.chdir(oe.data.expand(dldir, d))
219                         cvsroot = ":" + method + ":" + user
220                         if pswd:
221                                 cvsroot += ":" + pswd
222                         cvsroot += "@" + host + ":" + path
223
224                         cvscmd = "cvs -d" + cvsroot
225                         cvscmd += " checkout " + string.join(options) + " " + module 
226                         oe.note("fetch " + loc)
227                         oe.debug(1, "Running %s" % cvscmd)
228                         myret = os.system(cvscmd)
229                         os.chdir(olddir)
230                         if myret != 0:
231                                 raise FetchError(module)
232
233 methods.append(Cvs())
234
235 class Bk(Fetch):
236         def supports(url):
237                 """Check to see if a given url can be fetched via bitkeeper.
238                    Expects supplied url in list form, as outputted by oe.decodeurl().
239                 """
240                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
241                 return type in ['bk']
242         supports = staticmethod(supports)
243
244 methods.append(Bk())
245
246 class Local(Fetch):
247         def supports(url):
248                 """Check to see if a given url can be fetched in the local filesystem.
249                    Expects supplied url in list form, as outputted by oe.decodeurl().
250                 """
251                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
252                 return type in ['file','patch']
253         supports = staticmethod(supports)
254
255         def localpath(url):
256                 """Return the local filename of a given url assuming a successful fetch.
257                 """
258                 return url.split("://")[1]
259         localpath = staticmethod(localpath)
260
261         def go(self, urls = []):
262                 """Fetch urls (no-op for Local method)"""
263                 # no need to fetch local files, we'll deal with them in place.
264                 return 1
265
266 methods.append(Local())