Update TODO
[bitbake.git] / bin / oe / fetch.py
1 #!/usr/bin/python
2 """
3 OpenEmbedded 'Fetch' implementations
4
5 Classes for obtaining upstream sources for the
6 OpenEmbedded (http://openembedded.org) build infrastructure.
7
8 NOTE that it requires Python 2.x due to its use of static methods.
9
10 Copyright: (c) 2003 Chris Larson
11
12 Based on functions from the base oe module, Copyright 2003 Holger Schurig
13 """
14
15 import os, re, string
16 import oe
17 from oe import data
18
19 class FetchError(Exception):
20         """Exception raised when a download fails"""
21
22 class NoMethodError(Exception):
23         """Exception raised when there is no method to obtain a supplied url or set of urls"""
24
25 class MissingParameterError(Exception):
26         """Exception raised when a fetch method is missing a critical parameter in the url"""
27
28 methods = []
29
30 def init(urls = []):
31         for m in methods:
32                 m.urls = []
33
34         for u in urls:
35                 for m in methods:
36                         if m.supports(u):
37                                 m.urls.append(u)
38         
39 def go(d = data.init()):
40         """Fetch all urls"""
41         for m in methods:
42                 if m.urls:
43                         m.go(d)
44
45 def localpaths():
46         """Return a list of the local filenames, assuming successful fetch"""
47         local = []
48         for m in methods:
49                 for u in m.urls:
50                         local.append(m.localpath(u))
51         return local
52
53 def localpath(url):
54         for m in methods:
55                 if m.supports(url):
56                         return m.localpath(url)
57         return url 
58
59 class Fetch(object):
60         """Base class for 'fetch'ing data"""
61         
62         def __init__(self, urls = []):
63                 self.urls = []
64                 for url in urls:
65                         if self.supports(oe.decodeurl(url)) is 1:
66                                 self.urls.append(url)
67
68         def supports(url):
69                 """Check to see if this fetch class supports a given url.
70                    Expects supplied url in list form, as outputted by oe.decodeurl().
71                 """
72                 return 0
73         supports = staticmethod(supports)
74
75         def localpath(url):
76                 """Return the local filename of a given url assuming a successful fetch.
77                 """
78                 return url
79         localpath = staticmethod(localpath)
80
81         def setUrls(self, urls):
82                 self.__urls = urls
83
84         def getUrls(self):
85                 return self.__urls
86
87         urls = property(getUrls, setUrls, None, "Urls property")
88
89         def go(self, urls = []):
90                 """Fetch urls"""
91                 raise NoMethodError("Missing implementation for url")
92
93 class Wget(Fetch):
94         """Class to fetch urls via 'wget'"""
95         def supports(url):
96                 """Check to see if a given url can be fetched using wget.
97                    Expects supplied url in list form, as outputted by oe.decodeurl().
98                 """
99                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
100                 return type in ['http','https','ftp']
101         supports = staticmethod(supports)
102
103         def localpath(url):
104                 # strip off parameters
105                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
106                 if parm.has_key("localpath"):
107                         # if user overrides local path, use it.
108                         return parm["localpath"]
109                 url = oe.encodeurl([type, host, path, user, pswd, {}])
110                 return os.path.join(oe.getenv("DL_DIR"), os.path.basename(url))
111         localpath = staticmethod(localpath)
112
113         def go(self, d = data.init(), urls = []):
114                 """Fetch urls"""
115                 if not urls:
116                         urls = self.urls
117
118                 for loc in urls:
119                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(loc))
120                         myfile = os.path.basename(path)
121                         dlfile = self.localpath(loc)
122                         dlfile = data.expand(dlfile, d)
123
124                         if os.path.exists(dlfile):
125                                 # if the file exists, check md5
126                                 # if no md5 or mismatch, attempt resume.
127                                 # regardless of exit code, move on.
128                                 myfetch = data.expand(data.getVar("RESUMECOMMAND", d), d)
129                                 oe.note("fetch " +loc)
130                                 myfetch = myfetch.replace("${URI}",oe.encodeurl([type, host, path, user, pswd, {}]))
131                                 myfetch = myfetch.replace("${FILE}",myfile)
132                                 oe.debug(2,myfetch)
133                                 myret = os.system(myfetch)
134                                 continue
135                         else:
136                                 myfetch = data.expand(data.getVar("FETCHCOMMAND", d), d)
137                                 oe.note("fetch " +loc)
138                                 myfetch = myfetch.replace("${URI}",oe.encodeurl([type, host, path, user, pswd, {}]))
139                                 myfetch = myfetch.replace("${FILE}",myfile)
140                                 oe.debug(2,myfetch)
141                                 myret = os.system(myfetch)
142                                 if myret != 0:
143                                         raise FetchError(myfile)
144
145 methods.append(Wget())
146
147 class Cvs(Fetch):
148         """Class to fetch a module or modules from cvs repositories"""
149         checkoutopts = { "tag": "-r",
150                          "date": "-D" }
151
152         def supports(url):
153                 """Check to see if a given url can be fetched with cvs.
154                    Expects supplied url in list form, as outputted by oe.decodeurl().
155                 """
156                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
157                 return type in ['cvs', 'pserver']
158         supports = staticmethod(supports)
159
160         def localpath(url):
161                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
162                 if parm.has_key("localpath"):
163                         # if user overrides local path, use it.
164                         return parm["localpath"]
165
166                 if not parm.has_key("module"):
167                         return url
168                 else:
169                         return os.path.join(oe.getenv("DL_DIR"), parm["module"])
170         localpath = staticmethod(localpath)
171
172         def go(self, d = data.init(), urls = []):
173                 """Fetch urls"""
174                 if not urls:
175                         urls = self.urls
176
177                 for loc in urls:
178                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(loc))
179                         if not parm.has_key("module"):
180                                 raise MissingParameterError("cvs method needs a 'module' parameter")
181                         else:
182                                 module = parm["module"]
183
184                         dlfile = self.localpath(loc)
185                         # if local path contains the cvs
186                         # module, consider the dir above it to be the
187                         # download directory
188                         pos = dlfile.find(module)
189                         if pos:
190                                 dldir = dlfile[:pos]
191                         else:
192                                 dldir = os.path.dirname(dlfile)
193
194                         options = []
195
196                         for opt in self.checkoutopts:
197                                 if parm.has_key(opt):
198                                         options.append(self.checkoutopts[opt] + " " + parm[opt])
199
200                         if parm.has_key("method"):
201                                 method = parm["method"]
202                         else:
203                                 method = "pserver"
204
205                         os.chdir(data.expand(dldir, d))
206                         cvsroot = ":" + method + ":" + user
207                         if pswd:
208                                 cvsroot += ":" + pswd
209                         cvsroot += "@" + host + ":" + path
210
211 #                       if method == "pserver":
212 #                               # Login to the server
213 #                               cvscmd = "cvs -d" + cvsroot + " login"
214 #                               myret = os.system(cvscmd)
215 #                               if myret != 0:
216 #                                       raise FetchError(module)
217
218                         cvscmd = "cvs -d" + cvsroot
219                         cvscmd += " checkout " + string.join(options) + " " + module 
220                         oe.note("fetch " + loc)
221                         oe.debug(1, "Running %s" % cvscmd)
222                         myret = os.system(cvscmd)
223                         if myret != 0:
224                                 raise FetchError(module)
225
226 methods.append(Cvs())
227
228 class Bk(Fetch):
229         def supports(url):
230                 """Check to see if a given url can be fetched via bitkeeper.
231                    Expects supplied url in list form, as outputted by oe.decodeurl().
232                 """
233                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
234                 return type in ['bk']
235         supports = staticmethod(supports)
236
237 methods.append(Bk())
238
239 class Local(Fetch):
240         def supports(url):
241                 """Check to see if a given url can be fetched in the local filesystem.
242                    Expects supplied url in list form, as outputted by oe.decodeurl().
243                 """
244                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
245                 return type in ['file','patch']
246         supports = staticmethod(supports)
247
248         def localpath(url):
249                 """Return the local filename of a given url assuming a successful fetch.
250                 """
251                 return url.split("://")[1]
252         localpath = staticmethod(localpath)
253
254         def go(self, urls = []):
255                 """Fetch urls (no-op for Local method)"""
256                 # no need to fetch local files, we'll deal with them in place.
257                 return 1
258
259 methods.append(Local())