simplerss/src/httpclient.py

   1 from twisted.internet.protocol import ClientFactory
   2 from twisted.web2.client.http import HTTPClientProtocol
   3 from twisted.internet import reactor, error
   4 from urlparse import urlsplit
   5 from socket import gethostbyname
   6 from urllib import urlencode as urllib_urlencode, quote_plus as urllib_quote_plus
   7
   8 global HTTPCLIENT_requestCount
   9 HTTPCLIENT_requestCount = 0 # counts requests
  10
  11 class Enigma2HTTPRequest:
  12
  13     def __init__(self,hostname,path,port,method="GET",headerfields={}):
  14         self.hostname=hostname
  15         self.port=port
  16         self.path=path
  17         self.method=method
  18         self.headerfields = headerfields
  19         self.onRequestFinished = []
  20         self.onRequestError = []
  21         self.onHeaderLoaded = []
  22
  23         self.data=""
  24         self.readsize = 0
  25         self.headers= {}
  26
  27     def _DataRecived(self,data):
  28         self.readsize += len(data)
  29         self.data += data
  30
  31     def getIPAdress(self):
  32         """
  33             socket.gethostbyname() is syncron
  34             Enigma2 is blocked while process is running
  35         """
  36         try:
  37             return gethostbyname(self.hostname)
  38         except:
  39             return False
  40
  41     def HeaderLoaded(self,headers):
  42         self.headers = headers
  43         for i in self.onHeaderLoaded:
  44             if i is not None:
  45                 i(headers)
  46         self.onHeaderLoaded=[]
  47
  48     def RequestError(self,error):
  49         for i in self.onRequestError:
  50             if i is not None:
  51                 i(error)
  52         self.onRequestError = []
  53
  54     def RequestFinished(self,data):
  55         for i in self.onRequestFinished:
  56             if i is not None:
  57                 i(data)
  58
  59 class Enigma2URLHTTPRequest(Enigma2HTTPRequest):
  60     def __init__(self,url,method="GET",headerfields={}):
  61         x= urlsplit(url)
  62         if x[1].rfind(":")>0:
  63             y = x[1].split(":")
  64             hostname = y[0]
  65             port = int(y[1])
  66         else:
  67             hostname = x[1]
  68             port = 80
  69         path=x[2]
  70         Enigma2HTTPRequest.__init__(self,hostname,path,port,method=method,headerfields=headerfields)
  71
  72 class Enigma2FileHTTPRequest(Enigma2URLHTTPRequest):
  73     def __init__(self,targetfile,url,method="GET",headerfields={}):
  74         Enigma2URLHTTPRequest.__init__(self,url,method=method,headerfields=headerfields)
  75         self.filehandle = open(targetfile,"w")
  76         self.onRequestFinished.append(self.close)
  77         self.onRequestError.append(self.close)
  78     def close(self,dummy):
  79         self.filehandle.close()
  80
  81     def _DataRecived(self,data):
  82         self.readsize += len(data)
  83         self.filehandle.write(data)
  84
  85
  86
  87
  88 class Enigma2HTTPProtocol(HTTPClientProtocol):
  89     DEBUG = False
  90
  91     def __init__(self,request,requestCount):
  92         self.request = request
  93         self.requestCount = requestCount
  94         self.headers={}
  95         self.headerread=False
  96         self.responseFirstLine = True # to indikate, that first line of responseheader was read
  97         HTTPClientProtocol.__init__(self)
  98         self.setRawMode()
  99
 100     def rawDataReceived(self,line):
 101         for l in line.split(self.delimiter):
 102             if self.headerread:
 103                 self.request._DataRecived(l)
 104             else:
 105                 if self.DEBUG:
 106                     print "HTTP "+self.requestCount+" <<==",l
 107                 if l == "":
 108                     self.headerread = True
 109                     self.request.HeaderLoaded(self.headers)
 110                 else:
 111                     self.parseHeaderLine(l)
 112
 113     def parseHeaderLine(self,line):
 114         if self.responseFirstLine is  True:
 115             #print "parseHeaderLine",line.split(" ")
 116             fields = line.split(" ")
 117             protocoll = fields[0]
 118             responsecode  = fields[1]
 119             statuscode = " ".join(fields[2:])
 120             self.headers["protocoll"] = protocoll
 121             self.headers["responsecode"] = responsecode
 122             self.headers["statuscode"] = statuscode
 123             self.responseFirstLine = False
 124         elif line.rfind(":"):
 125             x = line.split(":")
 126             key = x[0].lstrip().rstrip().lower()
 127             var = ":".join(x[1:]).lstrip()
 128             self.headers[key] = var
 129         else:
 130             print "unknown headerline",line
 131
 132     def connectionMade(self):
 133         if self.request.method == "POST":
 134             (path,params ) = self.request.path.split("?")
 135         elif self.request.method == "GET":
 136             path = self.request.path
 137         self.sendLine("%s %s HTTP/1.0"%(self.request.method,path))
 138         self.sendLine("Host: %s"%self.request.hostname)
 139         for i in self.request.headerfields:
 140             self.sendLine(i+": "+self.request.headerfields[i])
 141         if self.request.method == "POST":
 142             self.sendLine("Content-Type: application/x-www-form-urlencoded")
 143             self.sendLine("Content-Length: "+str(len(params)))
 144
 145         self.sendLine("")
 146         if self.request.method == "POST":
 147             self.sendLine(params)
 148
 149     def sendLine(self,data):
 150         if self.DEBUG:
 151             print "HTTP "+self.requestCount+" ==>>",data
 152         HTTPClientProtocol.sendLine(self,data)
 153
 154 class Enigma2HTTPClientFactory(ClientFactory):
 155
 156     initialDelay = 20
 157     maxDelay = 500
 158     def __init__(self,request):
 159         self.hangup_ok = False
 160         self.request = request
 161
 162     def startedConnecting(self, connector):
 163         pass
 164
 165     def buildProtocol(self, addr):
 166         global HTTPCLIENT_requestCount
 167         HTTPCLIENT_requestCount = HTTPCLIENT_requestCount + 1
 168         return Enigma2HTTPProtocol(self.request,str(HTTPCLIENT_requestCount))
 169
 170     def clientConnectionLost(self, connector, reason):
 171         if not self.hangup_ok:
 172             self.request.RequestFinished(self.request.data)
 173         ClientFactory.clientConnectionLost(self, connector, reason)
 174
 175     def clientConnectionFailed(self, connector, reason):
 176         self.request.RequestError(reason.getErrorMessage())
 177         ClientFactory.clientConnectionFailed(self, connector, reason)
 178
 179 def urlencode(dict):
 180         return urllib_urlencode(dict)
 181
 182 def quote_plus(data):
 183         return urllib_quote_plus(data)
 184
 185 def getURL(url,callback=None,errorback=None,headercallback=None,method="GET",headers={}):
 186     """
 187         this will is called with a url
 188         url = http://www.hostna.me/somewhere/on/the/server <string>
 189     """
 190     req = Enigma2URLHTTPRequest(url,method=method,headerfields=headers)
 191     req.onRequestError.append(errorback)
 192     req.onHeaderLoaded.append(headercallback)
 193     req.onRequestFinished.append(callback)
 194     ipadress = req.getIPAdress()
 195     if ipadress is not False:
 196         reactor.connectTCP(ipadress,req.port, Enigma2HTTPClientFactory(req))
 197         return req
 198     else:
 199         if errorback is not None:
 200             errorback("Error while resolve Hostname")
 201
 202 def getPage(hostname,port,path,method="GET",callback=None,errorback=None,headercallback=None,headers={}):
 203     """
 204         this will is called with separte hostname,port,path
 205         hostname = www.hostna.me <string>
 206         port = 80 <int>
 207         path= /somewhere/on/the/server <string>
 208     """
 209     req = Enigma2HTTPRequest(hostname,path,port,method=method,headerfields=headers)
 210     req.onRequestError.append(errorback)
 211     req.onRequestFinished.append(callback)
 212     ipadress = req.getIPAdress()
 213     if ipadress is not False:
 214         reactor.connectTCP(ipadress,req.port, Enigma2HTTPClientFactory(req))
 215         return req
 216     else:
 217         if errorback is not None:
 218             errorback("Error while resolve Hostname")
 219
 220 def getFile(filename,url,method="GET",callback=None,errorback=None,headercallback=None,headers={}):
 221     """
 222         this will is called with a url and a target file
 223         fimename = /tmp/target.jpg
 224         url = http://www.hostna.me/somewhere/on/the/server.jpg <string>
 225     """
 226     req = Enigma2FileHTTPRequest(filename,url,method=method,headerfields=headers)
 227     req.onRequestError.append(errorback)
 228     req.onHeaderLoaded.append(headercallback)
 229     req.onRequestFinished.append(callback)
 230     ipadress = req.getIPAdress()
 231     if ipadress is not False:
 232         reactor.connectTCP(ipadress,req.port, Enigma2HTTPClientFactory(req))
 233         return req
 234     else:
 235         if errorback is not None:
 236             errorback("Error while resolve Hostname")
 237