bk cp oefetch.py oe/fetch.py
[bitbake.git] / bin / oe / fetch.py
1 """
2 OpenEmbedded 'Fetch' implementations
3
4 Classes for obtaining upstream sources for the
5 OpenEmbedded (http://openembedded.org) build infrastructure.
6
7 NOTE that it requires Python 2.x due to its use of static methods.
8
9 Copyright: (c) 2003 Chris Larson
10
11 Based on functions from the base oe module, Copyright 2003 Holger Schurig
12 """
13
14 import os, re
15 from oe import *
16
17 class FetchError(Exception):
18         """Exception raised when a download fails"""
19
20 class NoMethodError(Exception):
21         """Exception raised when there is no method to obtain a supplied url or set of urls"""
22
23 class MissingParameterError(Exception):
24         """Exception raised when a fetch method is missing a critical parameter in the url"""
25
26 methods = []
27
28 def init(urls = []):
29         for m in methods:
30                 m.urls = []
31
32         for u in urls:
33                 for m in methods:
34                         if m.supports(u):
35                                 m.urls.append(u)
36         
37 def go():
38         """Fetch all urls"""
39         for m in methods:
40                 m.go()
41
42 def localpaths():
43         """Return a list of the local filenames, assuming successful fetch"""
44         local = []
45         for m in methods:
46                 for u in m.urls:
47                         local.append(m.localpath(u))
48         return local
49
50 def localpath(url):
51         for m in methods:
52                 if m.supports(url):
53                         return m.localpath(url)
54         return url 
55
56 class Fetch(object):
57         """Base class for 'fetch'ing data"""
58         
59         def __init__(self, urls = []):
60                 self.urls = []
61                 for url in urls:
62                         if self.supports(decodeurl(url)) is 1:
63                                 self.urls.append(url)
64
65         def supports(url):
66                 """Check to see if this fetch class supports a given url.
67                    Expects supplied url in list form, as outputted by oe.decodeurl().
68                 """
69                 return 0
70         supports = staticmethod(supports)
71
72         def localpath(url):
73                 """Return the local filename of a given url assuming a successful fetch.
74                 """
75                 return url
76         localpath = staticmethod(localpath)
77
78         def setUrls(self, urls):
79                 self.__urls = urls
80
81         def getUrls(self):
82                 return self.__urls
83
84         urls = property(getUrls, setUrls, None, "Urls property")
85
86         def go(self, urls = []):
87                 """Fetch urls"""
88                 raise NoMethodError("Missing implementation for url")
89
90 class Wget(Fetch):
91         """Class to fetch urls via 'wget'"""
92         def supports(url):
93                 """Check to see if a given url can be fetched using wget.
94                    Expects supplied url in list form, as outputted by oe.decodeurl().
95                 """
96                 (type, host, path, user, pswd, parm) = decodeurl(expand(url))
97                 return type in ['http','https','ftp']
98         supports = staticmethod(supports)
99
100         def localpath(url):
101                 # strip off parameters
102                 (type, host, path, user, pswd, parm) = decodeurl(expand(url))
103                 if parm.has_key("localpath"):
104                         # if user overrides local path, use it.
105                         return parm["localpath"]
106                 url = encodeurl([type, host, path, user, pswd, {}])
107                 return os.path.join(getenv("DL_DIR"), os.path.basename(url))
108         localpath = staticmethod(localpath)
109
110         def go(self, urls = []):
111                 """Fetch urls"""
112                 if not urls:
113                         urls = self.urls
114
115                 for loc in urls:
116                         (type, host, path, user, pswd, parm) = decodeurl(expand(loc))
117                         myfile = os.path.basename(path)
118                         dlfile = self.localpath(loc)
119
120                         myfetch = getenv("RESUMECOMMAND")
121                         note("fetch " +loc)
122                         myfetch = myfetch.replace("${URI}",encodeurl([type, host, path, user, pswd, {}]))
123                         myfetch = myfetch.replace("${FILE}",myfile)
124                         debug(2,myfetch)
125                         myret = os.system(myfetch)
126                         if myret != 0:
127                                 raise FetchError(myfile)
128
129 methods.append(Wget())
130
131 class Cvs(Fetch):
132         """Class to fetch a module or modules from cvs repositories"""
133         checkoutopts = { "tag": "-r",
134                          "date": "-D" }
135
136         def supports(url):
137                 """Check to see if a given url can be fetched with cvs.
138                    Expects supplied url in list form, as outputted by oe.decodeurl().
139                 """
140                 (type, host, path, user, pswd, parm) = decodeurl(expand(url))
141                 return type in ['cvs', 'pserver']
142         supports = staticmethod(supports)
143
144         def localpath(url):
145                 (type, host, path, user, pswd, parm) = decodeurl(expand(url))
146                 if parm.has_key("localpath"):
147                         # if user overrides local path, use it.
148                         return parm["localpath"]
149
150                 if not parm.has_key("module"):
151                         return url
152                 else:
153                         return os.path.join(getenv("DL_DIR"), parm["module"])
154         localpath = staticmethod(localpath)
155
156         def go(self, urls = []):
157                 """Fetch urls"""
158                 if not urls:
159                         urls = self.urls
160
161                 for loc in urls:
162                         (type, host, path, user, pswd, parm) = decodeurl(expand(loc))
163                         if not parm.has_key("module"):
164                                 raise MissingParameterError("cvs method needs a 'module' parameter")
165                         else:
166                                 module = parm["module"]
167
168                         dlfile = self.localpath(loc)
169                         # if local path contains the cvs
170                         # module, consider the dir above it to be the
171                         # download directory
172                         pos = dlfile.find(module)
173                         if pos:
174                                 dldir = dlfile[:pos]
175                         else:
176                                 dldir = os.path.dirname(dlfile)
177
178                         options = []
179
180                         for opt in self.checkoutopts:
181                                 if parm.has_key(opt):
182                                         options.append(self.checkoutopts[opt] + " " + parm[opt])
183
184                         if parm.has_key("method"):
185                                 method = parm["method"]
186                         else:
187                                 method = "pserver"
188
189                         os.chdir(expand(dldir))
190                         cvsroot = ":" + method + ":" + user
191                         if pswd is not None:
192                                 cvsroot += ":" + pswd
193                         cvsroot += "@" + host + ":" + path
194
195 #                       if method == "pserver":
196 #                               # Login to the server
197 #                               cvscmd = "cvs -d" + cvsroot + " login"
198 #                               myret = os.system(cvscmd)
199 #                               if myret != 0:
200 #                                       raise FetchError(module)
201
202                         cvscmd = "cvs -d" + cvsroot
203                         cvscmd += " checkout " + string.join(options) + " " + module 
204                         note("fetch " + loc)
205                         myret = os.system(cvscmd)
206                         if myret != 0:
207                                 raise FetchError(module)
208
209 methods.append(Cvs())
210
211 class Bk(Fetch):
212         def supports(url):
213                 """Check to see if a given url can be fetched via bitkeeper.
214                    Expects supplied url in list form, as outputted by oe.decodeurl().
215                 """
216                 (type, host, path, user, pswd, parm) = decodeurl(expand(url))
217                 return type in ['bk']
218         supports = staticmethod(supports)
219
220 methods.append(Bk())
221
222 class Local(Fetch):
223         def supports(url):
224                 """Check to see if a given url can be fetched in the local filesystem.
225                    Expects supplied url in list form, as outputted by oe.decodeurl().
226                 """
227                 (type, host, path, user, pswd, parm) = decodeurl(expand(url))
228                 return type in ['file','patch']
229         supports = staticmethod(supports)
230
231         def localpath(url):
232                 """Return the local filename of a given url assuming a successful fetch.
233                 """
234                 return url.split("://")[1]
235         localpath = staticmethod(localpath)
236
237         def go(self, urls = []):
238                 """Fetch urls (no-op for Local method)"""
239                 # no need to fetch local files, we'll deal with them in place.
240                 return 1
241
242 methods.append(Local())