Return to previous dir after fetching
[bitbake.git] / bin / oe / fetch.py
1 #!/usr/bin/python
2 """
3 OpenEmbedded 'Fetch' implementations
4
5 Classes for obtaining upstream sources for the
6 OpenEmbedded (http://openembedded.org) build infrastructure.
7
8 NOTE that it requires Python 2.x due to its use of static methods.
9
10 Copyright: (c) 2003 Chris Larson
11
12 Based on functions from the base oe module, Copyright 2003 Holger Schurig
13 """
14
15 import os, re, string
16 import oe
17 from oe import data
18
19 class FetchError(Exception):
20         """Exception raised when a download fails"""
21
22 class NoMethodError(Exception):
23         """Exception raised when there is no method to obtain a supplied url or set of urls"""
24
25 class MissingParameterError(Exception):
26         """Exception raised when a fetch method is missing a critical parameter in the url"""
27
28 methods = []
29
30 def init(urls = []):
31         for m in methods:
32                 m.urls = []
33
34         for u in urls:
35                 for m in methods:
36                         if m.supports(u):
37                                 m.urls.append(u)
38         
39 def go(d = data.init()):
40         """Fetch all urls"""
41         for m in methods:
42                 if m.urls:
43                         m.go(d)
44
45 def localpaths():
46         """Return a list of the local filenames, assuming successful fetch"""
47         local = []
48         for m in methods:
49                 for u in m.urls:
50                         local.append(m.localpath(u))
51         return local
52
53 def localpath(url):
54         for m in methods:
55                 if m.supports(url):
56                         return m.localpath(url)
57         return url 
58
59 class Fetch(object):
60         """Base class for 'fetch'ing data"""
61         
62         def __init__(self, urls = []):
63                 self.urls = []
64                 for url in urls:
65                         if self.supports(oe.decodeurl(url)) is 1:
66                                 self.urls.append(url)
67
68         def supports(url):
69                 """Check to see if this fetch class supports a given url.
70                    Expects supplied url in list form, as outputted by oe.decodeurl().
71                 """
72                 return 0
73         supports = staticmethod(supports)
74
75         def localpath(url):
76                 """Return the local filename of a given url assuming a successful fetch.
77                 """
78                 return url
79         localpath = staticmethod(localpath)
80
81         def setUrls(self, urls):
82                 self.__urls = urls
83
84         def getUrls(self):
85                 return self.__urls
86
87         urls = property(getUrls, setUrls, None, "Urls property")
88
89         def go(self, urls = []):
90                 """Fetch urls"""
91                 raise NoMethodError("Missing implementation for url")
92
93 class Wget(Fetch):
94         """Class to fetch urls via 'wget'"""
95         def supports(url):
96                 """Check to see if a given url can be fetched using wget.
97                    Expects supplied url in list form, as outputted by oe.decodeurl().
98                 """
99                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
100                 return type in ['http','https','ftp']
101         supports = staticmethod(supports)
102
103         def localpath(url):
104                 # strip off parameters
105                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
106                 if parm.has_key("localpath"):
107                         # if user overrides local path, use it.
108                         return parm["localpath"]
109                 url = oe.encodeurl([type, host, path, user, pswd, {}])
110                 return os.path.join(oe.getenv("DL_DIR"), os.path.basename(url))
111         localpath = staticmethod(localpath)
112
113         def go(self, d = data.init(), urls = []):
114                 """Fetch urls"""
115                 if not urls:
116                         urls = self.urls
117
118                 for loc in urls:
119                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(loc))
120                         myfile = os.path.basename(path)
121                         dlfile = self.localpath(loc)
122                         dlfile = data.expand(dlfile, d)
123
124                         if os.path.exists(dlfile):
125                                 # if the file exists, check md5
126                                 # if no md5 or mismatch, attempt resume.
127                                 # regardless of exit code, move on.
128                                 myfetch = data.expand(data.getVar("RESUMECOMMAND", d), d)
129                                 oe.note("fetch " +loc)
130                                 myfetch = myfetch.replace("${URI}",oe.encodeurl([type, host, path, user, pswd, {}]))
131                                 myfetch = myfetch.replace("${FILE}",myfile)
132                                 oe.debug(2,myfetch)
133                                 myret = os.system(myfetch)
134                                 continue
135                         else:
136                                 myfetch = data.expand(data.getVar("FETCHCOMMAND", d), d)
137                                 oe.note("fetch " +loc)
138                                 myfetch = myfetch.replace("${URI}",oe.encodeurl([type, host, path, user, pswd, {}]))
139                                 myfetch = myfetch.replace("${FILE}",myfile)
140                                 oe.debug(2,myfetch)
141                                 myret = os.system(myfetch)
142                                 if myret != 0:
143                                         raise FetchError(myfile)
144
145 methods.append(Wget())
146
147 class Cvs(Fetch):
148         """Class to fetch a module or modules from cvs repositories"""
149         checkoutopts = { "tag": "-r",
150                          "date": "-D" }
151
152         def supports(url):
153                 """Check to see if a given url can be fetched with cvs.
154                    Expects supplied url in list form, as outputted by oe.decodeurl().
155                 """
156                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
157                 return type in ['cvs', 'pserver']
158         supports = staticmethod(supports)
159
160         def localpath(url):
161                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
162                 if parm.has_key("localpath"):
163                         # if user overrides local path, use it.
164                         return parm["localpath"]
165
166                 if not parm.has_key("module"):
167                         return url
168                 else:
169                         return os.path.join(oe.getenv("DL_DIR"), parm["module"])
170         localpath = staticmethod(localpath)
171
172         def go(self, d = data.init(), urls = []):
173                 """Fetch urls"""
174                 if not urls:
175                         urls = self.urls
176
177                 for loc in urls:
178                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(loc))
179                         if not parm.has_key("module"):
180                                 raise MissingParameterError("cvs method needs a 'module' parameter")
181                         else:
182                                 module = parm["module"]
183
184                         dlfile = self.localpath(loc)
185                         # if local path contains the cvs
186                         # module, consider the dir above it to be the
187                         # download directory
188                         pos = dlfile.find(module)
189                         if pos:
190                                 dldir = dlfile[:pos]
191                         else:
192                                 dldir = os.path.dirname(dlfile)
193
194                         options = []
195
196                         for opt in self.checkoutopts:
197                                 if parm.has_key(opt):
198                                         options.append(self.checkoutopts[opt] + " " + parm[opt])
199
200                         if parm.has_key("method"):
201                                 method = parm["method"]
202                         else:
203                                 method = "pserver"
204
205                         olddir = os.path.abspath(os.getcwd())
206                         os.chdir(data.expand(dldir, d))
207                         cvsroot = ":" + method + ":" + user
208                         if pswd:
209                                 cvsroot += ":" + pswd
210                         cvsroot += "@" + host + ":" + path
211
212 #                       if method == "pserver":
213 #                               # Login to the server
214 #                               cvscmd = "cvs -d" + cvsroot + " login"
215 #                               myret = os.system(cvscmd)
216 #                               if myret != 0:
217 #                                       raise FetchError(module)
218
219                         cvscmd = "cvs -d" + cvsroot
220                         cvscmd += " checkout " + string.join(options) + " " + module 
221                         oe.note("fetch " + loc)
222                         oe.debug(1, "Running %s" % cvscmd)
223                         myret = os.system(cvscmd)
224                         os.chdir(olddir)
225                         if myret != 0:
226                                 raise FetchError(module)
227
228 methods.append(Cvs())
229
230 class Bk(Fetch):
231         def supports(url):
232                 """Check to see if a given url can be fetched via bitkeeper.
233                    Expects supplied url in list form, as outputted by oe.decodeurl().
234                 """
235                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
236                 return type in ['bk']
237         supports = staticmethod(supports)
238
239 methods.append(Bk())
240
241 class Local(Fetch):
242         def supports(url):
243                 """Check to see if a given url can be fetched in the local filesystem.
244                    Expects supplied url in list form, as outputted by oe.decodeurl().
245                 """
246                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
247                 return type in ['file','patch']
248         supports = staticmethod(supports)
249
250         def localpath(url):
251                 """Return the local filename of a given url assuming a successful fetch.
252                 """
253                 return url.split("://")[1]
254         localpath = staticmethod(localpath)
255
256         def go(self, urls = []):
257                 """Fetch urls (no-op for Local method)"""
258                 # no need to fetch local files, we'll deal with them in place.
259                 return 1
260
261 methods.append(Local())