(no commit message)
[bitbake.git] / bin / oe / fetch.py
1 """
2 OpenEmbedded 'Fetch' implementations
3
4 Classes for obtaining upstream sources for the
5 OpenEmbedded (http://openembedded.org) build infrastructure.
6
7 NOTE that it requires Python 2.x due to its use of static methods.
8
9 Copyright: (c) 2003 Chris Larson
10
11 Based on functions from the base oe module, Copyright 2003 Holger Schurig
12 """
13
14 import os, re, string
15 import oe
16 import oe.data
17
18 class FetchError(Exception):
19         """Exception raised when a download fails"""
20
21 class NoMethodError(Exception):
22         """Exception raised when there is no method to obtain a supplied url or set of urls"""
23
24 class MissingParameterError(Exception):
25         """Exception raised when a fetch method is missing a critical parameter in the url"""
26
27 methods = []
28
29 def init(urls = []):
30         for m in methods:
31                 m.urls = []
32
33         for u in urls:
34                 for m in methods:
35                         if m.supports(u):
36                                 m.urls.append(u)
37         
38 def go():
39         """Fetch all urls"""
40         for m in methods:
41                 if m.urls:
42                         m.go()
43
44 def localpaths():
45         """Return a list of the local filenames, assuming successful fetch"""
46         local = []
47         for m in methods:
48                 for u in m.urls:
49                         local.append(m.localpath(u))
50         return local
51
52 def localpath(url):
53         for m in methods:
54                 if m.supports(url):
55                         return m.localpath(url)
56         return url 
57
58 class Fetch(object):
59         """Base class for 'fetch'ing data"""
60         
61         def __init__(self, urls = []):
62                 self.urls = []
63                 for url in urls:
64                         if self.supports(oe.decodeurl(url)) is 1:
65                                 self.urls.append(url)
66
67         def supports(url):
68                 """Check to see if this fetch class supports a given url.
69                    Expects supplied url in list form, as outputted by oe.decodeurl().
70                 """
71                 return 0
72         supports = staticmethod(supports)
73
74         def localpath(url):
75                 """Return the local filename of a given url assuming a successful fetch.
76                 """
77                 return url
78         localpath = staticmethod(localpath)
79
80         def setUrls(self, urls):
81                 self.__urls = urls
82
83         def getUrls(self):
84                 return self.__urls
85
86         urls = property(getUrls, setUrls, None, "Urls property")
87
88         def go(self, urls = []):
89                 """Fetch urls"""
90                 raise NoMethodError("Missing implementation for url")
91
92 class Wget(Fetch):
93         """Class to fetch urls via 'wget'"""
94         def supports(url):
95                 """Check to see if a given url can be fetched using wget.
96                    Expects supplied url in list form, as outputted by oe.decodeurl().
97                 """
98                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
99                 return type in ['http','https','ftp']
100         supports = staticmethod(supports)
101
102         def localpath(url):
103                 # strip off parameters
104                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
105                 if parm.has_key("localpath"):
106                         # if user overrides local path, use it.
107                         return parm["localpath"]
108                 url = oe.encodeurl([type, host, path, user, pswd, {}])
109                 return os.path.join(oe.getenv("DL_DIR"), os.path.basename(url))
110         localpath = staticmethod(localpath)
111
112         def go(self, urls = []):
113                 """Fetch urls"""
114                 if not urls:
115                         urls = self.urls
116
117                 for loc in urls:
118                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(loc))
119                         myfile = os.path.basename(path)
120                         dlfile = self.localpath(loc)
121
122                         myfetch = oe.getenv("RESUMECOMMAND")
123                         oe.note("fetch " +loc)
124                         myfetch = myfetch.replace("${URI}",oe.encodeurl([type, host, path, user, pswd, {}]))
125                         myfetch = myfetch.replace("${FILE}",myfile)
126                         oe.debug(2,myfetch)
127                         myret = os.system(myfetch)
128                         if myret != 0:
129                                 raise FetchError(myfile)
130
131 methods.append(Wget())
132
133 class Cvs(Fetch):
134         """Class to fetch a module or modules from cvs repositories"""
135         checkoutopts = { "tag": "-r",
136                          "date": "-D" }
137
138         def supports(url):
139                 """Check to see if a given url can be fetched with cvs.
140                    Expects supplied url in list form, as outputted by oe.decodeurl().
141                 """
142                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
143                 return type in ['cvs', 'pserver']
144         supports = staticmethod(supports)
145
146         def localpath(url):
147                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
148                 if parm.has_key("localpath"):
149                         # if user overrides local path, use it.
150                         return parm["localpath"]
151
152                 if not parm.has_key("module"):
153                         return url
154                 else:
155                         return os.path.join(oe.getenv("DL_DIR"), parm["module"])
156         localpath = staticmethod(localpath)
157
158         def go(self, urls = []):
159                 """Fetch urls"""
160                 if not urls:
161                         urls = self.urls
162
163                 for loc in urls:
164                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(loc))
165                         if not parm.has_key("module"):
166                                 raise MissingParameterError("cvs method needs a 'module' parameter")
167                         else:
168                                 module = parm["module"]
169
170                         dlfile = self.localpath(loc)
171                         # if local path contains the cvs
172                         # module, consider the dir above it to be the
173                         # download directory
174                         pos = dlfile.find(module)
175                         if pos:
176                                 dldir = dlfile[:pos]
177                         else:
178                                 dldir = os.path.dirname(dlfile)
179
180                         options = []
181
182                         for opt in self.checkoutopts:
183                                 if parm.has_key(opt):
184                                         options.append(self.checkoutopts[opt] + " " + parm[opt])
185
186                         if parm.has_key("method"):
187                                 method = parm["method"]
188                         else:
189                                 method = "pserver"
190
191                         os.chdir(oe.expand(dldir))
192                         cvsroot = ":" + method + ":" + user
193                         if pswd is not None:
194                                 cvsroot += ":" + pswd
195                         cvsroot += "@" + host + ":" + path
196
197 #                       if method == "pserver":
198 #                               # Login to the server
199 #                               cvscmd = "cvs -d" + cvsroot + " login"
200 #                               myret = os.system(cvscmd)
201 #                               if myret != 0:
202 #                                       raise FetchError(module)
203
204                         cvscmd = "cvs -d" + cvsroot
205                         cvscmd += " checkout " + string.join(options) + " " + module 
206                         oe.note("fetch " + loc)
207                         myret = os.system(cvscmd)
208                         if myret != 0:
209                                 raise FetchError(module)
210
211 methods.append(Cvs())
212
213 class Bk(Fetch):
214         def supports(url):
215                 """Check to see if a given url can be fetched via bitkeeper.
216                    Expects supplied url in list form, as outputted by oe.decodeurl().
217                 """
218                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
219                 return type in ['bk']
220         supports = staticmethod(supports)
221
222 methods.append(Bk())
223
224 class Local(Fetch):
225         def supports(url):
226                 """Check to see if a given url can be fetched in the local filesystem.
227                    Expects supplied url in list form, as outputted by oe.decodeurl().
228                 """
229                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.expand(url))
230                 return type in ['file','patch']
231         supports = staticmethod(supports)
232
233         def localpath(url):
234                 """Return the local filename of a given url assuming a successful fetch.
235                 """
236                 return url.split("://")[1]
237         localpath = staticmethod(localpath)
238
239         def go(self, urls = []):
240                 """Fetch urls (no-op for Local method)"""
241                 # no need to fetch local files, we'll deal with them in place.
242                 return 1
243
244 methods.append(Local())