implemented searching on page
[enigma2-plugins.git] / mytube / src / MyTubeService.py
1 # -*- coding: iso-8859-1 -*-
2 from enigma import ePythonMessagePump
3
4 from __init__ import decrypt_block
5 from ThreadQueue import ThreadQueue
6 import gdata.youtube
7 import gdata.youtube.service
8 from gdata.service import BadAuthentication
9
10 from twisted.web import client
11 from twisted.internet import reactor
12 from urllib2 import Request, URLError, urlopen as urlopen2
13 from socket import gaierror, error
14 import os, socket
15 from urllib import quote, unquote_plus, unquote
16 from httplib import HTTPConnection, CannotSendRequest, BadStatusLine, HTTPException
17
18 from urlparse import parse_qs
19 from threading import Thread
20
21 HTTPConnection.debuglevel = 1
22
23 def validate_cert(cert, key):
24         buf = decrypt_block(cert[8:], key)
25         if buf is None:
26                 return None
27         return buf[36:107] + cert[139:196]
28
29 def get_rnd():
30         try:
31                 rnd = os.urandom(8)
32                 return rnd
33         except:
34                 return None
35
36 std_headers = {
37         'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.6) Gecko/20100627 Firefox/3.6.6',
38         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
39         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
40         'Accept-Language': 'en-us,en;q=0.5',
41 }
42
43 #config.plugins.mytube = ConfigSubsection()
44 #config.plugins.mytube.general = ConfigSubsection()
45 #config.plugins.mytube.general.useHTTPProxy = ConfigYesNo(default = False)
46 #config.plugins.mytube.general.ProxyIP = ConfigIP(default=[0,0,0,0])
47 #config.plugins.mytube.general.ProxyPort = ConfigNumber(default=8080)
48 #class MyOpener(FancyURLopener):
49 #       version = 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12'
50
51
52 class GoogleSuggestions():
53         def __init__(self):
54                 self.hl = "en"
55                 self.conn = None
56
57         def prepareQuery(self):
58                 #GET /complete/search?output=toolbar&client=youtube-psuggest&xml=true&ds=yt&hl=en&jsonp=self.gotSuggestions&q=s
59                 self.prepQuerry = "/complete/search?output=toolbar&client=youtube&xml=true&ds=yt&"
60                 if self.hl is not None:
61                         self.prepQuerry = self.prepQuerry + "hl=" + self.hl + "&"
62                 self.prepQuerry = self.prepQuerry + "jsonp=self.gotSuggestions&q="
63                 print "[MyTube - GoogleSuggestions] prepareQuery:",self.prepQuerry
64
65         def getSuggestions(self, queryString):
66                 self.prepareQuery()
67                 if queryString is not "":
68                         query = self.prepQuerry + quote(queryString)
69                         self.conn = HTTPConnection("google.com")
70                         try:
71                                 self.conn = HTTPConnection("google.com")
72                                 self.conn.request("GET", query, "", {"Accept-Encoding": "UTF-8"})
73                         except (CannotSendRequest, gaierror, error):
74                                 self.conn.close()
75                                 print "[MyTube - GoogleSuggestions] Can not send request for suggestions"
76                                 return None
77                         else:
78                                 try:
79                                         response = self.conn.getresponse()
80                                 except BadStatusLine:
81                                         self.conn.close()
82                                         print "[MyTube - GoogleSuggestions] Can not get a response from google"
83                                         return None
84                                 else:
85                                         if response.status == 200:
86                                                 data = response.read()
87                                                 header = response.getheader("Content-Type", "text/xml; charset=ISO-8859-1")
88                                                 charset = "ISO-8859-1"
89                                                 try:
90                                                         charset = header.split(";")[1].split("=")[1]
91                                                         print "[MyTube - GoogleSuggestions] Got charset %s" %charset
92                                                 except:
93                                                         print "[MyTube - GoogleSuggestions] No charset in Header, falling back to %s" %charset
94                                                 data = data.decode(charset).encode("utf-8")
95                                                 self.conn.close()
96                                                 return data
97                                         else:
98                                                 self.conn.close()
99                                                 return None
100                 else:
101                         return None
102
103 class MyTubeFeedEntry():
104         def __init__(self, feed, entry, favoritesFeed = False):
105                 self.feed = feed
106                 self.entry = entry
107                 self.favoritesFeed = favoritesFeed
108                 self.thumbnail = {}
109                 """self.myopener = MyOpener()
110                 urllib.urlopen = MyOpener().open
111                 if config.plugins.mytube.general.useHTTPProxy.value is True:
112                         proxy = {'http': 'http://'+str(config.plugins.mytube.general.ProxyIP.getText())+':'+str(config.plugins.mytube.general.ProxyPort.value)}
113                         self.myopener = MyOpener(proxies=proxy)
114                         urllib.urlopen = MyOpener(proxies=proxy).open
115                 else:
116                         self.myopener = MyOpener()
117                         urllib.urlopen = MyOpener().open"""
118
119         def isPlaylistEntry(self):
120                 return False
121
122         def getTubeId(self):
123                 #print "[MyTubeFeedEntry] getTubeId"
124                 ret = None
125                 if self.entry.media.player:
126                         split = self.entry.media.player.url.split("=")
127                         ret = split.pop()
128                         if ret.startswith('youtube_gdata'):
129                                 tmpval=split.pop()
130                                 if tmpval.endswith("&feature"):
131                                         tmp = tmpval.split("&")
132                                         ret = tmp.pop(0)
133                 return ret
134
135         def getTitle(self):
136                 #print "[MyTubeFeedEntry] getTitle",self.entry.media.title.text
137                 return self.entry.media.title.text
138
139         def getDescription(self):
140                 #print "[MyTubeFeedEntry] getDescription"
141                 if self.entry.media is not None and self.entry.media.description is not None:
142                         return self.entry.media.description.text
143                 return "not vailable"
144
145         def getThumbnailUrl(self, index = 0):
146                 #print "[MyTubeFeedEntry] getThumbnailUrl"
147                 if index < len(self.entry.media.thumbnail):
148                         return self.entry.media.thumbnail[index].url
149                 return None
150
151         def getPublishedDate(self):
152                 if self.entry.published is not None:
153                         return self.entry.published.text
154                 return "unknown"
155
156         def getViews(self):
157                 if self.entry.statistics is not None:
158                         return self.entry.statistics.view_count
159                 return "not available"
160
161         def getDuration(self):
162                 if self.entry.media is not None and self.entry.media.duration is not None:
163                         return self.entry.media.duration.seconds
164                 else:
165                         return 0
166
167         def getRatingAverage(self):
168                 if self.entry.rating is not None:
169                         return self.entry.rating.average
170                 return 0
171
172
173         def getNumRaters(self):
174                 if self.entry.rating is not None:
175                         return self.entry.rating.num_raters
176                 return ""
177
178         def getAuthor(self):
179                 authors = []
180                 for author in self.entry.author:
181                         authors.append(author.name.text)
182                 author = ", ".join(authors)
183                 return author
184
185         def PrintEntryDetails(self):
186                 EntryDetails = { 'Title': None, 'TubeID': None, 'Published': None, 'Published': None, 'Description': None, 'Category': None, 'Tags': None, 'Duration': None, 'Views': None, 'Rating': None, 'Thumbnails': None}
187                 EntryDetails['Title'] = self.entry.media.title.text
188                 EntryDetails['TubeID'] = self.getTubeId()
189                 EntryDetails['Description'] = self.getDescription()
190                 EntryDetails['Category'] = self.entry.media.category[0].text
191                 EntryDetails['Tags'] = self.entry.media.keywords.text
192                 EntryDetails['Published'] = self.getPublishedDate()
193                 EntryDetails['Views'] = self.getViews()
194                 EntryDetails['Duration'] = self.getDuration()
195                 EntryDetails['Rating'] = self.getNumRaters()
196                 EntryDetails['RatingAverage'] = self.getRatingAverage()
197                 EntryDetails['Author'] = self.getAuthor()
198                 # show thumbnails
199                 list = []
200                 for thumbnail in self.entry.media.thumbnail:
201                         print 'Thumbnail url: %s' % thumbnail.url
202                         list.append(str(thumbnail.url))
203                 EntryDetails['Thumbnails'] = list
204                 #print EntryDetails
205                 return EntryDetails
206
207         def getVideoUrl(self):
208                 VIDEO_FMT_PRIORITY_MAP = {
209                         '38' : 1, #MP4 Original (HD)
210                         '37' : 2, #MP4 1080p (HD)
211                         '22' : 3, #MP4 720p (HD)
212                         '18' : 4, #MP4 360p
213                         '35' : 5, #FLV 480p
214                         '34' : 6, #FLV 360p
215                 }
216                 video_url = None
217                 video_id = str(self.getTubeId())
218
219                 # Getting video webpage
220                 #URLs for YouTube video pages will change from the format http://www.youtube.com/watch?v=ylLzyHk54Z0 to http://www.youtube.com/watch#!v=ylLzyHk54Z0.
221                 watch_url = 'http://www.youtube.com/watch?v=%s&gl=US&hl=en' % video_id
222                 watchrequest = Request(watch_url, None, std_headers)
223                 try:
224                         print "[MyTube] trying to find out if a HD Stream is available",watch_url
225                         watchvideopage = urlopen2(watchrequest).read()
226                 except (URLError, HTTPException, socket.error), err:
227                         print "[MyTube] Error: Unable to retrieve watchpage - Error code: ", str(err)
228                         return video_url
229
230                 # Get video info
231                 for el in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
232                         info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en' % (video_id, el))
233                         request = Request(info_url, None, std_headers)
234                         try:
235                                 infopage = urlopen2(request).read()
236                                 videoinfo = parse_qs(infopage)
237                                 if ('url_encoded_fmt_stream_map' or 'fmt_url_map') in videoinfo:
238                                         break
239                         except (URLError, HTTPException, socket.error), err:
240                                 print "[MyTube] Error: unable to download video infopage",str(err)
241                                 return video_url
242
243                 if ('url_encoded_fmt_stream_map' or 'fmt_url_map') not in videoinfo:
244                         # Attempt to see if YouTube has issued an error message
245                         if 'reason' not in videoinfo:
246                                 print '[MyTube] Error: unable to extract "fmt_url_map" or "url_encoded_fmt_stream_map" parameter for unknown reason'
247                         else:
248                                 reason = unquote_plus(videoinfo['reason'][0])
249                                 print '[MyTube] Error: YouTube said: %s' % reason.decode('utf-8')
250                         return video_url
251
252                 video_fmt_map = {}
253                 fmt_infomap = {}
254                 if videoinfo.has_key('url_encoded_fmt_stream_map'):
255                         tmp_fmtUrlDATA = videoinfo['url_encoded_fmt_stream_map'][0].split(',')
256                 else:
257                         tmp_fmtUrlDATA = videoinfo['fmt_url_map'][0].split(',')
258                 for fmtstring in tmp_fmtUrlDATA:
259                         fmturl = fmtid = fmtsig = ""
260                         if videoinfo.has_key('url_encoded_fmt_stream_map'):
261                                 try:
262                                         for arg in fmtstring.split('&'):
263                                                 if arg.find('=') >= 0:
264                                                         print arg.split('=')
265                                                         key, value = arg.split('=')
266                                                         if key == 'itag':
267                                                                 if len(value) > 3:
268                                                                         value = value[:2]
269                                                                 fmtid = value
270                                                         elif key == 'url':
271                                                                 fmturl = value
272                                                         elif key == 'sig':
273                                                                 fmtsig = value
274                                                                 
275                                         if fmtid != "" and fmturl != "" and fmtsig != ""  and VIDEO_FMT_PRIORITY_MAP.has_key(fmtid):
276                                                 video_fmt_map[VIDEO_FMT_PRIORITY_MAP[fmtid]] = { 'fmtid': fmtid, 'fmturl': unquote_plus(fmturl), 'fmtsig': fmtsig }
277                                                 fmt_infomap[int(fmtid)] = "%s&signature=%s" %(unquote_plus(fmturl), fmtsig)
278                                         fmturl = fmtid = fmtsig = ""
279
280                                 except:
281                                         print "error parsing fmtstring:",fmtstring
282                                         
283                         else:
284                                 (fmtid,fmturl) = fmtstring.split('|')
285                         if VIDEO_FMT_PRIORITY_MAP.has_key(fmtid) and fmtid != "":
286                                 video_fmt_map[VIDEO_FMT_PRIORITY_MAP[fmtid]] = { 'fmtid': fmtid, 'fmturl': unquote_plus(fmturl) }
287                                 fmt_infomap[int(fmtid)] = unquote_plus(fmturl)
288                 print "[MyTube] got",sorted(fmt_infomap.iterkeys())
289                 if video_fmt_map and len(video_fmt_map):
290                         print "[MyTube] found best available video format:",video_fmt_map[sorted(video_fmt_map.iterkeys())[0]]['fmtid']
291                         best_video = video_fmt_map[sorted(video_fmt_map.iterkeys())[0]]
292                         video_url = "%s&signature=%s" %(best_video['fmturl'].split(';')[0], best_video['fmtsig'])
293                         print "[MyTube] found best available video url:",video_url
294
295                 return video_url
296
297         def getRelatedVideos(self):
298                 print "[MyTubeFeedEntry] getRelatedVideos()"
299                 for link in self.entry.link:
300                         #print "Related link: ", link.rel.endswith
301                         if link.rel.endswith("video.related"):
302                                 print "Found Related: ", link.href
303                                 return link.href
304
305         def getResponseVideos(self):
306                 print "[MyTubeFeedEntry] getResponseVideos()"
307                 for link in self.entry.link:
308                         #print "Responses link: ", link.rel.endswith
309                         if link.rel.endswith("video.responses"):
310                                 print "Found Responses: ", link.href
311                                 return link.href
312
313         def getUserVideos(self):
314                 print "[MyTubeFeedEntry] getUserVideos()"
315                 username = self.getAuthor()
316                 myuri = 'http://gdata.youtube.com/feeds/api/users/%s/uploads' % username
317                 print "Found Uservideos: ", myuri
318                 return myuri
319
320 class MyTubePlayerService():
321 #       Do not change the client_id and developer_key in the login-section!
322 #       ClientId: ytapi-dream-MyTubePlayer-i0kqrebg-0
323 #       DeveloperKey: AI39si4AjyvU8GoJGncYzmqMCwelUnqjEMWTFCcUtK-VUzvWygvwPO-sadNwW5tNj9DDCHju3nnJEPvFy4WZZ6hzFYCx8rJ6Mw
324         def __init__(self):
325                 print "[MyTube] MyTubePlayerService - init"
326                 self.feedentries = []
327                 self.feed = None
328
329         def startService(self):
330                 print "[MyTube] MyTubePlayerService - startService"
331                 self.yt_service = gdata.youtube.service.YouTubeService(
332                         developer_key = 'AI39si4AjyvU8GoJGncYzmqMCwelUnqjEMWTFCcUtK-VUzvWygvwPO-sadNwW5tNj9DDCHju3nnJEPvFy4WZZ6hzFYCx8rJ6Mw',
333                         client_id = 'ytapi-dream-MyTubePlayer-i0kqrebg-0'
334                 )
335 #               self.loggedIn = False
336                 #os.environ['http_proxy'] = 'http://169.229.50.12:3128'
337                 #proxy = os.environ.get('http_proxy')
338                 #print "FOUND ENV PROXY-->",proxy
339                 #for a in os.environ.keys():
340                 #       print a
341
342         def stopService(self):
343                 print "[MyTube] MyTubePlayerService - stopService"
344                 del self.ytService
345
346         def getFeedService(self, feedname):
347                 if feedname == "top_rated":
348                         return self.yt_service.GetTopRatedVideoFeed
349                 elif feedname == "most_viewed":
350                         return self.yt_service.GetMostViewedVideoFeed
351                 elif feedname == "recently_featured":
352                         return self.yt_service.GetRecentlyFeaturedVideoFeed
353                 elif feedname == "top_favorites":
354                         return self.yt_service.GetTopFavoritesVideoFeed
355                 elif feedname == "most_recent":
356                         return self.yt_service.GetMostRecentVideoFeed
357                 elif feedname == "most_discussed":
358                         return self.yt_service.GetMostDiscussedVideoFeed
359                 elif feedname == "most_linked":
360                         return self.yt_service.GetMostLinkedVideoFeed
361                 elif feedname == "most_responded":
362                         return self.yt_service.GetMostRespondedVideoFeed
363                 return self.yt_service.GetYouTubeVideoFeed
364
365         def getFeed(self, url, feedname = "", callback = None, errorback = None):
366                 print "[MyTube] MyTubePlayerService - getFeed:",url, feedname
367                 self.feedentries = []
368                 ytservice = self.yt_service.GetYouTubeVideoFeed
369                 if feedname in ("hd", "most_popular", "most_shared", "on_the_web"):
370                         if feedname == "hd":
371                                 url = "http://gdata.youtube.com/feeds/api/videos/-/HD"
372                         else:
373                                 url = url + feedname
374                 elif feedname in ("top_rated","most_viewed","recently_featured","top_favorites","most_recent","most_discussed","most_linked","most_responded"):
375                         url = None
376                         ytservice = self.getFeedService(feedname)
377
378                 queryThread = YoutubeQueryThread(ytservice, url, self.gotFeed, self.gotFeedError, callback, errorback)
379                 queryThread.start()
380                 return queryThread
381
382         def search(self, searchTerms, startIndex = 1, maxResults = 25,
383                                         orderby = "relevance", time = 'all_time', racy = "include",
384                                         author = "", lr = "", categories = "", sortOrder = "ascending",
385                                         callback = None, errorback = None):
386                 print "[MyTube] MyTubePlayerService - search()"
387                 self.feedentries = []
388                 query = gdata.youtube.service.YouTubeVideoQuery()
389                 query.vq = searchTerms
390                 query.orderby = orderby
391                 query.time = time
392                 query.racy = racy
393                 query.sortorder = sortOrder
394                 if lr is not None:
395                         query.lr = lr
396                 if categories[0] is not None:
397                         query.categories = categories
398                 query.start_index = startIndex
399                 query.max_results = maxResults
400                 queryThread = YoutubeQueryThread(self.yt_service.YouTubeQuery, query, self.gotFeed, self.gotFeedError, callback, errorback)
401                 queryThread.start()
402                 return queryThread
403
404         def gotFeed(self, feed, callback):
405                 if feed is not None:
406                         self.feed = feed
407                         for entry in self.feed.entry:
408                                 MyFeedEntry = MyTubeFeedEntry(self, entry)
409                                 self.feedentries.append(MyFeedEntry)
410                 if callback is not None:
411                         callback(self.feed)
412
413         def gotFeedError(self, exception, errorback):
414                 if errorback is not None:
415                         errorback(exception)
416
417
418         def getTitle(self):
419                 return self.feed.title.text
420
421         def getEntries(self):
422                 return self.feedentries
423
424         def itemCount(self):
425                 return self.feed.items_per_page.text
426
427         def getTotalResults(self):
428                 return self.feed.total_results.text
429
430         def getNextFeedEntriesURL(self):
431                 for link in self.feed.link:
432                         if link.rel == "next":
433                                 return link.href
434                 return None
435
436         def getCurrentPage(self):
437                 return int(int(self.feed.start_index.text) / int(self.itemCount())) + 1
438
439 class YoutubeQueryThread(Thread):
440         def __init__(self, query, param, gotFeed, gotFeedError, callback, errorback):
441                 Thread.__init__(self)
442                 self.messagePump = ePythonMessagePump()
443                 self.messages = ThreadQueue()
444                 self.gotFeed = gotFeed
445                 self.gotFeedError = gotFeedError
446                 self.callback = callback
447                 self.errorback = errorback
448                 self.query = query
449                 self.param = param
450                 self.canceled = False
451                 self.messagePump.recv_msg.get().append(self.finished)
452
453         def cancel(self):
454                 self.canceled = True
455
456         def run(self):
457                 try:
458                         if self.param is None:
459                                 feed = self.query()
460                         else:
461                                 feed = self.query(self.param)
462                         self.messages.push((True, feed, self.callback))
463                         self.messagePump.send(0)
464                 except Exception, ex:
465                         self.messages.push((False, ex, self.errorback))
466                         self.messagePump.send(0)
467
468         def finished(self, val):
469                 if not self.canceled:
470                         message = self.messages.pop()
471                         if message[0]:
472                                 self.gotFeed(message[1], message[2])
473                         else:
474                                 self.gotFeedError(message[1], message[2])
475
476 myTubeService = MyTubePlayerService()