Enhancements all around, rework of the python function implementation, move do_fetch...
[bitbake.git] / bin / oe / fetch.py
1 """
2 OpenEmbedded 'Fetch' implementations
3
4 Classes for obtaining upstream sources for the
5 OpenEmbedded (http://openembedded.org) build infrastructure.
6
7 NOTE that it requires Python 2.x due to its use of static methods.
8
9 Copyright: (c) 2003 Chris Larson
10
11 Based on functions from the base oe module, Copyright 2003 Holger Schurig
12 """
13
14 import os, re, string
15 import oe
16 from oe import data
17
18 class FetchError(Exception):
19         """Exception raised when a download fails"""
20
21 class NoMethodError(Exception):
22         """Exception raised when there is no method to obtain a supplied url or set of urls"""
23
24 class MissingParameterError(Exception):
25         """Exception raised when a fetch method is missing a critical parameter in the url"""
26
27 methods = []
28
29 def init(urls = []):
30         for m in methods:
31                 m.urls = []
32
33         for u in urls:
34                 for m in methods:
35                         if m.supports(u):
36                                 m.urls.append(u)
37         
38 def go(d = data.init()):
39         """Fetch all urls"""
40         for m in methods:
41                 if m.urls:
42                         m.go(d)
43
44 def localpaths():
45         """Return a list of the local filenames, assuming successful fetch"""
46         local = []
47         for m in methods:
48                 for u in m.urls:
49                         local.append(m.localpath(u))
50         return local
51
52 def localpath(url):
53         for m in methods:
54                 if m.supports(url):
55                         return m.localpath(url)
56         return url 
57
58 class Fetch(object):
59         """Base class for 'fetch'ing data"""
60         
61         def __init__(self, urls = []):
62                 self.urls = []
63                 for url in urls:
64                         if self.supports(oe.decodeurl(url)) is 1:
65                                 self.urls.append(url)
66
67         def supports(url):
68                 """Check to see if this fetch class supports a given url.
69                    Expects supplied url in list form, as outputted by oe.decodeurl().
70                 """
71                 return 0
72         supports = staticmethod(supports)
73
74         def localpath(url):
75                 """Return the local filename of a given url assuming a successful fetch.
76                 """
77                 return url
78         localpath = staticmethod(localpath)
79
80         def setUrls(self, urls):
81                 self.__urls = urls
82
83         def getUrls(self):
84                 return self.__urls
85
86         urls = property(getUrls, setUrls, None, "Urls property")
87
88         def go(self, urls = []):
89                 """Fetch urls"""
90                 raise NoMethodError("Missing implementation for url")
91
92 class Wget(Fetch):
93         """Class to fetch urls via 'wget'"""
94         def supports(url):
95                 """Check to see if a given url can be fetched using wget.
96                    Expects supplied url in list form, as outputted by oe.decodeurl().
97                 """
98                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
99                 return type in ['http','https','ftp']
100         supports = staticmethod(supports)
101
102         def localpath(url):
103                 # strip off parameters
104                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
105                 if parm.has_key("localpath"):
106                         # if user overrides local path, use it.
107                         return parm["localpath"]
108                 url = oe.encodeurl([type, host, path, user, pswd, {}])
109                 return os.path.join(oe.getenv("DL_DIR"), os.path.basename(url))
110         localpath = staticmethod(localpath)
111
112         def go(self, d = data.init(), urls = []):
113                 """Fetch urls"""
114                 if not urls:
115                         urls = self.urls
116
117                 for loc in urls:
118                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(loc))
119                         myfile = os.path.basename(path)
120                         dlfile = self.localpath(loc)
121                         dlfile = data.expand(dlfile, d)
122
123                         if os.path.exists(dlfile):
124                                 # if the file exists, check md5
125                                 # if no md5 or mismatch, attempt resume.
126                                 # regardless of exit code, move on.
127                                 myfetch = data.expand(data.getVar("RESUMECOMMAND", d), d)
128                                 oe.note("fetch " +loc)
129                                 myfetch = myfetch.replace("${URI}",oe.encodeurl([type, host, path, user, pswd, {}]))
130                                 myfetch = myfetch.replace("${FILE}",myfile)
131                                 oe.debug(2,myfetch)
132                                 myret = os.system(myfetch)
133                                 continue
134                         else:
135                                 myfetch = data.expand(data.getVar("FETCHCOMMAND", d), d)
136                                 oe.note("fetch " +loc)
137                                 myfetch = myfetch.replace("${URI}",oe.encodeurl([type, host, path, user, pswd, {}]))
138                                 myfetch = myfetch.replace("${FILE}",myfile)
139                                 oe.debug(2,myfetch)
140                                 myret = os.system(myfetch)
141                                 if myret != 0:
142                                         raise FetchError(myfile)
143
144 methods.append(Wget())
145
146 class Cvs(Fetch):
147         """Class to fetch a module or modules from cvs repositories"""
148         checkoutopts = { "tag": "-r",
149                          "date": "-D" }
150
151         def supports(url):
152                 """Check to see if a given url can be fetched with cvs.
153                    Expects supplied url in list form, as outputted by oe.decodeurl().
154                 """
155                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
156                 return type in ['cvs', 'pserver']
157         supports = staticmethod(supports)
158
159         def localpath(url):
160                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
161                 if parm.has_key("localpath"):
162                         # if user overrides local path, use it.
163                         return parm["localpath"]
164
165                 if not parm.has_key("module"):
166                         return url
167                 else:
168                         return os.path.join(oe.getenv("DL_DIR"), parm["module"])
169         localpath = staticmethod(localpath)
170
171         def go(self, d = data.init(), urls = []):
172                 """Fetch urls"""
173                 if not urls:
174                         urls = self.urls
175
176                 for loc in urls:
177                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(loc))
178                         if not parm.has_key("module"):
179                                 raise MissingParameterError("cvs method needs a 'module' parameter")
180                         else:
181                                 module = parm["module"]
182
183                         dlfile = self.localpath(loc)
184                         # if local path contains the cvs
185                         # module, consider the dir above it to be the
186                         # download directory
187                         pos = dlfile.find(module)
188                         if pos:
189                                 dldir = dlfile[:pos]
190                         else:
191                                 dldir = os.path.dirname(dlfile)
192
193                         options = []
194
195                         for opt in self.checkoutopts:
196                                 if parm.has_key(opt):
197                                         options.append(self.checkoutopts[opt] + " " + parm[opt])
198
199                         if parm.has_key("method"):
200                                 method = parm["method"]
201                         else:
202                                 method = "pserver"
203
204                         os.chdir(data.expand(dldir, d))
205                         cvsroot = ":" + method + ":" + user
206                         if pswd:
207                                 cvsroot += ":" + pswd
208                         cvsroot += "@" + host + ":" + path
209
210 #                       if method == "pserver":
211 #                               # Login to the server
212 #                               cvscmd = "cvs -d" + cvsroot + " login"
213 #                               myret = os.system(cvscmd)
214 #                               if myret != 0:
215 #                                       raise FetchError(module)
216
217                         cvscmd = "cvs -d" + cvsroot
218                         cvscmd += " checkout " + string.join(options) + " " + module 
219                         oe.note("fetch " + loc)
220                         oe.debug(1, "Running %s" % cvscmd)
221                         myret = os.system(cvscmd)
222                         if myret != 0:
223                                 raise FetchError(module)
224
225 methods.append(Cvs())
226
227 class Bk(Fetch):
228         def supports(url):
229                 """Check to see if a given url can be fetched via bitkeeper.
230                    Expects supplied url in list form, as outputted by oe.decodeurl().
231                 """
232                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
233                 return type in ['bk']
234         supports = staticmethod(supports)
235
236 methods.append(Bk())
237
238 class Local(Fetch):
239         def supports(url):
240                 """Check to see if a given url can be fetched in the local filesystem.
241                    Expects supplied url in list form, as outputted by oe.decodeurl().
242                 """
243                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
244                 return type in ['file','patch']
245         supports = staticmethod(supports)
246
247         def localpath(url):
248                 """Return the local filename of a given url assuming a successful fetch.
249                 """
250                 return url.split("://")[1]
251         localpath = staticmethod(localpath)
252
253         def go(self, urls = []):
254                 """Fetch urls (no-op for Local method)"""
255                 # no need to fetch local files, we'll deal with them in place.
256                 return 1
257
258 methods.append(Local())