Merge branch 'master' of ssh://scm.schwerkraft.elitedvb.net/scmrepos/git/enigma2...
[enigma2-plugins.git] / mytube / src / MyTubeService.py
1 # -*- coding: iso-8859-1 -*-
2 from enigma import ePythonMessagePump
3
4 from __init__ import decrypt_block
5 from ThreadQueue import ThreadQueue
6 import gdata.youtube
7 import gdata.youtube.service
8 from gdata.service import BadAuthentication
9
10 from youtube_dl import YoutubeDL
11
12 from twisted.web import client
13 from twisted.internet import reactor
14 from urllib2 import Request, URLError, urlopen as urlopen2
15 from socket import gaierror, error
16 import os, socket, httplib
17 from urllib import quote, unquote_plus, unquote, urlencode
18 from httplib import HTTPConnection, CannotSendRequest, BadStatusLine, HTTPException
19
20 from urlparse import parse_qs, parse_qsl
21 from threading import Thread
22
23 HTTPConnection.debuglevel = 1
24
25 if 'HTTPSConnection' not in dir(httplib):
26         # python on enimga2 has no https socket support
27         gdata.youtube.service.YOUTUBE_USER_FEED_URI = 'http://gdata.youtube.com/feeds/api/users'
28
29 def validate_cert(cert, key):
30         buf = decrypt_block(cert[8:], key)
31         if buf is None:
32                 return None
33         return buf[36:107] + cert[139:196]
34
35 def get_rnd():
36         try:
37                 rnd = os.urandom(8)
38                 return rnd
39         except:
40                 return None
41
42 std_headers = {
43         'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.6) Gecko/20100627 Firefox/3.6.6',
44         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
45         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
46         'Accept-Language': 'en-us,en;q=0.5',
47 }
48
49 #config.plugins.mytube = ConfigSubsection()
50 #config.plugins.mytube.general = ConfigSubsection()
51 #config.plugins.mytube.general.useHTTPProxy = ConfigYesNo(default = False)
52 #config.plugins.mytube.general.ProxyIP = ConfigIP(default=[0,0,0,0])
53 #config.plugins.mytube.general.ProxyPort = ConfigNumber(default=8080)
54 #class MyOpener(FancyURLopener):
55 #       version = 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12'
56
57
58 class GoogleSuggestions():
59         def __init__(self):
60                 self.hl = "en"
61                 self.conn = None
62
63         def prepareQuery(self):
64                 #GET /complete/search?output=toolbar&client=youtube-psuggest&xml=true&ds=yt&hl=en&jsonp=self.gotSuggestions&q=s
65                 #self.prepQuerry = "/complete/search?output=toolbar&client=youtube&xml=true&ds=yt&"
66                 self.prepQuerry = "/complete/search?output=chrome&client=chrome&"
67                 if self.hl is not None:
68                         self.prepQuerry = self.prepQuerry + "hl=" + self.hl + "&"
69                 self.prepQuerry = self.prepQuerry + "jsonp=self.gotSuggestions&q="
70                 print "[MyTube - GoogleSuggestions] prepareQuery:",self.prepQuerry
71
72         def getSuggestions(self, queryString):
73                 self.prepareQuery()
74                 if queryString is not "":
75                         query = self.prepQuerry + quote(queryString)
76                         self.conn = HTTPConnection("google.com")
77                         try:
78                                 self.conn = HTTPConnection("google.com")
79                                 self.conn.request("GET", query, "", {"Accept-Encoding": "UTF-8"})
80                         except (CannotSendRequest, gaierror, error):
81                                 self.conn.close()
82                                 print "[MyTube - GoogleSuggestions] Can not send request for suggestions"
83                                 return None
84                         else:
85                                 try:
86                                         response = self.conn.getresponse()
87                                 except BadStatusLine:
88                                         self.conn.close()
89                                         print "[MyTube - GoogleSuggestions] Can not get a response from google"
90                                         return None
91                                 else:
92                                         if response.status == 200:
93                                                 data = response.read()
94                                                 header = response.getheader("Content-Type", "text/xml; charset=ISO-8859-1")
95                                                 charset = "ISO-8859-1"
96                                                 try:
97                                                         charset = header.split(";")[1].split("=")[1]
98                                                         print "[MyTube - GoogleSuggestions] Got charset %s" %charset
99                                                 except:
100                                                         print "[MyTube - GoogleSuggestions] No charset in Header, falling back to %s" %charset
101                                                 data = data.decode(charset).encode("utf-8")
102                                                 self.conn.close()
103                                                 return data
104                                         else:
105                                                 self.conn.close()
106                                                 return None
107                 else:
108                         return None
109
110 class MyTubeFeedEntry():
111         def __init__(self, feed, entry, favoritesFeed = False):
112                 self.feed = feed
113                 self.entry = entry
114                 self.favoritesFeed = favoritesFeed
115                 self.thumbnail = {}
116                 """self.myopener = MyOpener()
117                 urllib.urlopen = MyOpener().open
118                 if config.plugins.mytube.general.useHTTPProxy.value is True:
119                         proxy = {'http': 'http://'+str(config.plugins.mytube.general.ProxyIP.getText())+':'+str(config.plugins.mytube.general.ProxyPort.value)}
120                         self.myopener = MyOpener(proxies=proxy)
121                         urllib.urlopen = MyOpener(proxies=proxy).open
122                 else:
123                         self.myopener = MyOpener()
124                         urllib.urlopen = MyOpener().open"""
125
126         def isPlaylistEntry(self):
127                 return False
128
129         def getTubeId(self):
130                 #print "[MyTubeFeedEntry] getTubeId"
131                 ret = None
132                 if self.entry.media.player:
133                         split = self.entry.media.player.url.split("=")
134                         ret = split.pop()
135                         if ret.startswith('youtube_gdata'):
136                                 tmpval=split.pop()
137                                 if tmpval.endswith("&feature"):
138                                         tmp = tmpval.split("&")
139                                         ret = tmp.pop(0)
140                 return ret
141
142         def getTitle(self):
143                 #print "[MyTubeFeedEntry] getTitle",self.entry.media.title.text
144                 return self.entry.media.title.text
145
146         def getDescription(self):
147                 #print "[MyTubeFeedEntry] getDescription"
148                 if self.entry.media is not None and self.entry.media.description is not None:
149                         return self.entry.media.description.text
150                 return "not vailable"
151
152         def getThumbnailUrl(self, index = 0):
153                 #print "[MyTubeFeedEntry] getThumbnailUrl"
154                 if index < len(self.entry.media.thumbnail):
155                         return self.entry.media.thumbnail[index].url
156                 return None
157
158         def getPublishedDate(self):
159                 if self.entry.published is not None:
160                         return self.entry.published.text
161                 return "unknown"
162
163         def getViews(self):
164                 if self.entry.statistics is not None:
165                         return self.entry.statistics.view_count
166                 return "not available"
167
168         def getDuration(self):
169                 if self.entry.media is not None and self.entry.media.duration is not None:
170                         return self.entry.media.duration.seconds
171                 else:
172                         return 0
173
174         def getRatingAverage(self):
175                 if self.entry.rating is not None:
176                         return self.entry.rating.average
177                 return 0
178
179
180         def getNumRaters(self):
181                 if self.entry.rating is not None:
182                         return self.entry.rating.num_raters
183                 return ""
184
185         def getAuthor(self):
186                 authors = []
187                 for author in self.entry.author:
188                         authors.append(author.name.text)
189                 author = ", ".join(authors)
190                 return author
191
192         def getUserFeedsUrl(self):
193                 for author in self.entry.author:
194                         return author.uri.text
195
196                 return False
197
198         def getUserId(self):
199                 return self.getUserFeedsUrl().split('/')[-1]
200
201         def subscribeToUser(self):
202                 username = self.getUserId()
203                 return myTubeService.SubscribeToUser(username)
204                 
205         def addToFavorites(self):
206                 video_id = self.getTubeId()
207                 return myTubeService.addToFavorites(video_id)
208
209         def PrintEntryDetails(self):
210                 EntryDetails = { 'Title': None, 'TubeID': None, 'Published': None, 'Published': None, 'Description': None, 'Category': None, 'Tags': None, 'Duration': None, 'Views': None, 'Rating': None, 'Thumbnails': None}
211                 EntryDetails['Title'] = self.entry.media.title.text
212                 EntryDetails['TubeID'] = self.getTubeId()
213                 EntryDetails['Description'] = self.getDescription()
214                 EntryDetails['Category'] = self.entry.media.category[0].text
215                 EntryDetails['Tags'] = self.entry.media.keywords.text
216                 EntryDetails['Published'] = self.getPublishedDate()
217                 EntryDetails['Views'] = self.getViews()
218                 EntryDetails['Duration'] = self.getDuration()
219                 EntryDetails['Rating'] = self.getNumRaters()
220                 EntryDetails['RatingAverage'] = self.getRatingAverage()
221                 EntryDetails['Author'] = self.getAuthor()
222                 # show thumbnails
223                 list = []
224                 for thumbnail in self.entry.media.thumbnail:
225                         print 'Thumbnail url: %s' % thumbnail.url
226                         list.append(str(thumbnail.url))
227                 EntryDetails['Thumbnails'] = list
228                 #print EntryDetails
229                 return EntryDetails
230
231         def getVideoUrl(self):
232                 VIDEO_FMT_PRIORITY_MAP = {
233                         1 : '38', #MP4 Original (HD)
234                         2 : '37', #MP4 1080p (HD)
235                         3 : '22', #MP4 720p (HD)
236                         4 : '18', #MP4 360p
237                         5 : '35', #FLV 480p
238                         6 : '34', #FLV 360p
239                 }
240                 KEY_FORMAT_ID = u"format_id"
241                 KEY_URL = u"url"
242                 KEY_ENTRIES = u"entries"
243                 KEY_FORMATS = u"formats"
244
245                 video_url = None
246                 video_id = str(self.getTubeId())
247
248                 # Getting video webpage
249                 #URLs for YouTube video pages will change from the format http://www.youtube.com/watch?v=ylLzyHk54Z0 to http://www.youtube.com/watch#!v=ylLzyHk54Z0.
250                 watch_url = 'http://www.youtube.com/watch?v=%s' % video_id
251                 format_prio = "/".join(VIDEO_FMT_PRIORITY_MAP.itervalues())
252                 ytdl = YoutubeDL(params={"youtube_include_dash_manifest": False, "format" : format_prio})
253                 result = ytdl.extract_info(watch_url, download=False)
254                 if KEY_ENTRIES in result: # Can be a playlist or a list of videos
255                         entry = result[KEY_ENTRIES][0] #TODO handle properly
256                 else:# Just a video
257                         entry = result
258
259                 video_url = entry.get(KEY_URL)
260                 return str(video_url)
261
262         def getRelatedVideos(self):
263                 print "[MyTubeFeedEntry] getRelatedVideos()"
264                 for link in self.entry.link:
265                         #print "Related link: ", link.rel.endswith
266                         if link.rel.endswith("video.related"):
267                                 print "Found Related: ", link.href
268                                 return link.href
269
270         def getResponseVideos(self):
271                 print "[MyTubeFeedEntry] getResponseVideos()"
272                 for link in self.entry.link:
273                         #print "Responses link: ", link.rel.endswith
274                         if link.rel.endswith("video.responses"):
275                                 print "Found Responses: ", link.href
276                                 return link.href
277
278         def getUserVideos(self):
279                 print "[MyTubeFeedEntry] getUserVideos()"
280                 username = self.getUserId()
281                 myuri = 'http://gdata.youtube.com/feeds/api/users/%s/uploads' % username
282                 print "Found Uservideos: ", myuri
283                 return myuri
284
285 class MyTubePlayerService():
286 #       Do not change the client_id and developer_key in the login-section!
287 #       ClientId: ytapi-dream-MyTubePlayer-i0kqrebg-0
288 #       DeveloperKey: AI39si4AjyvU8GoJGncYzmqMCwelUnqjEMWTFCcUtK-VUzvWygvwPO-sadNwW5tNj9DDCHju3nnJEPvFy4WZZ6hzFYCx8rJ6Mw
289
290         cached_auth_request = {}
291         current_auth_token = None
292         yt_service = None
293
294         def __init__(self):
295                 print "[MyTube] MyTubePlayerService - init"
296                 self.feedentries = []
297                 self.feed = None
298
299         def startService(self):
300                 print "[MyTube] MyTubePlayerService - startService"
301
302                 self.yt_service = gdata.youtube.service.YouTubeService()
303
304                 # missing ssl support? youtube will help us on some feed urls
305                 self.yt_service.ssl = self.supportsSSL()
306
307                 # dont use it on class init; error on post and auth
308                 self.yt_service.developer_key = 'AI39si4AjyvU8GoJGncYzmqMCwelUnqjEMWTFCcUtK-VUzvWygvwPO-sadNwW5tNj9DDCHju3nnJEPvFy4WZZ6hzFYCx8rJ6Mw'
309                 self.yt_service.client_id = 'ytapi-dream-MyTubePlayer-i0kqrebg-0'
310
311                 # yt_service is reinit on every feed build; cache here to not reauth. remove init every time?
312                 if self.current_auth_token is not None:
313                         print "[MyTube] MyTubePlayerService - auth_cached"
314                         self.yt_service.SetClientLoginToken(self.current_auth_token)
315                 
316 #               self.loggedIn = False
317                 #os.environ['http_proxy'] = 'http://169.229.50.12:3128'
318                 #proxy = os.environ.get('http_proxy')
319                 #print "FOUND ENV PROXY-->",proxy
320                 #for a in os.environ.keys():
321                 #       print a
322
323         def stopService(self):
324                 print "[MyTube] MyTubePlayerService - stopService"
325                 del self.ytService
326
327         def getLoginTokenOnCurl(self, email, pw):
328
329                 opts = {
330                   'service':'youtube',
331                   'accountType': 'HOSTED_OR_GOOGLE',
332                   'Email': email,
333                   'Passwd': pw,
334                   'source': self.yt_service.client_id,
335                 }
336                 
337                 print "[MyTube] MyTubePlayerService - Starting external curl auth request"
338                 result = os.popen('curl -s -k -X POST "%s" -d "%s"' % (gdata.youtube.service.YOUTUBE_CLIENTLOGIN_AUTHENTICATION_URL , urlencode(opts))).read()
339                 
340                 return result
341
342         def supportsSSL(self):
343                 return 'HTTPSConnection' in dir(httplib)
344
345         def getFormattedTokenRequest(self, email, pw):
346                 return dict(parse_qsl(self.getLoginTokenOnCurl(email, pw).strip().replace('\n', '&')))
347         
348         def getAuthedUsername(self):
349                 # on external curl we can get real username
350                 if self.cached_auth_request.get('YouTubeUser') is not None:
351                         return self.cached_auth_request.get('YouTubeUser')
352
353                 if self.is_auth() is False:
354                         return ''
355
356                 # current gdata auth class save doesnt save realuser
357                 return 'Logged In'
358
359         def auth_user(self, username, password):
360                 print "[MyTube] MyTubePlayerService - auth_use - " + str(username)
361
362                 if self.yt_service is None:
363                         self.startService()
364                 
365                 if self.current_auth_token is not None:
366                         print "[MyTube] MyTubePlayerService - auth_cached"
367                         self.yt_service.SetClientLoginToken(self.current_auth_token)
368                         return
369
370                 if self.supportsSSL() is False:
371                         print "[MyTube] MyTubePlayerService - HTTPSConnection not found trying external curl"
372                         self.cached_auth_request = self.getFormattedTokenRequest(username, password)
373                         if self.cached_auth_request.get('Auth') is None:
374                                 raise Exception('Got no auth token from curl; you need curl and valid youtube login data')
375                         
376                         self.yt_service.SetClientLoginToken(self.cached_auth_request.get('Auth'))
377                 else:
378                         print "[MyTube] MyTubePlayerService - Using regularly ProgrammaticLogin for login"
379                         self.yt_service.email = username
380                         self.yt_service.password  = password
381                         self.yt_service.ProgrammaticLogin()
382                         
383                 # double check login: reset any token on wrong logins
384                 if self.is_auth() is False:
385                         print "[MyTube] MyTubePlayerService - auth_use - auth not possible resetting"
386                         self.resetAuthState()
387                         return
388
389                 print "[MyTube] MyTubePlayerService - Got successful login"
390                 self.current_auth_token = self.auth_token()
391
392         def resetAuthState(self):
393                 print "[MyTube] MyTubePlayerService - resetting auth"
394                 self.cached_auth_request = {}
395                 self.current_auth_token = None
396
397                 if self.yt_service is None:
398                         return
399
400                 self.yt_service.current_token = None
401                 self.yt_service.token_store.remove_all_tokens()
402
403         def is_auth(self):
404                 if self.current_auth_token is not None:
405                         return True             
406                 
407                 if self.yt_service.current_token is None:
408                         return False
409                 
410                 return self.yt_service.current_token.get_token_string() != 'None'
411
412         def auth_token(self):
413                 return self.yt_service.current_token.get_token_string()
414
415         def getFeedService(self, feedname):
416                 if feedname == "top_rated":
417                         return self.yt_service.GetTopRatedVideoFeed
418                 elif feedname == "most_viewed":
419                         return self.yt_service.GetMostViewedVideoFeed
420                 elif feedname == "recently_featured":
421                         return self.yt_service.GetRecentlyFeaturedVideoFeed
422                 elif feedname == "top_favorites":
423                         return self.yt_service.GetTopFavoritesVideoFeed
424                 elif feedname == "most_recent":
425                         return self.yt_service.GetMostRecentVideoFeed
426                 elif feedname == "most_discussed":
427                         return self.yt_service.GetMostDiscussedVideoFeed
428                 elif feedname == "most_linked":
429                         return self.yt_service.GetMostLinkedVideoFeed
430                 elif feedname == "most_responded":
431                         return self.yt_service.GetMostRespondedVideoFeed
432                 return self.yt_service.GetYouTubeVideoFeed
433
434         def getFeed(self, url, feedname = "", callback = None, errorback = None):
435                 print "[MyTube] MyTubePlayerService - getFeed:",url, feedname
436                 self.feedentries = []
437                 ytservice = self.yt_service.GetYouTubeVideoFeed
438                 
439                 if feedname == "my_subscriptions":
440                         url = "http://gdata.youtube.com/feeds/api/users/default/newsubscriptionvideos"
441                 elif feedname == "my_favorites":
442                         url = "http://gdata.youtube.com/feeds/api/users/default/favorites"
443                 elif feedname == "my_history":
444                         url = "http://gdata.youtube.com/feeds/api/users/default/watch_history?v=2"
445                 elif feedname == "my_recommendations":
446                         url = "http://gdata.youtube.com/feeds/api/users/default/recommendations?v=2"
447                 elif feedname == "my_watch_later":
448                         url = "http://gdata.youtube.com/feeds/api/users/default/watch_later?v=2"
449                 elif feedname == "my_uploads":
450                         url = "http://gdata.youtube.com/feeds/api/users/default/uploads"
451                 elif feedname in ("hd", "most_popular", "most_shared", "on_the_web"):
452                         if feedname == "hd":
453                                 url = "http://gdata.youtube.com/feeds/api/videos/-/HD"
454                         else:
455                                 url = url + feedname
456                 elif feedname in ("top_rated","most_viewed","recently_featured","top_favorites","most_recent","most_discussed","most_linked","most_responded"):
457                         url = None
458                         ytservice = self.getFeedService(feedname)
459
460                 queryThread = YoutubeQueryThread(ytservice, url, self.gotFeed, self.gotFeedError, callback, errorback)
461                 queryThread.start()
462                 return queryThread
463
464         def search(self, searchTerms, startIndex = 1, maxResults = 25,
465                                         orderby = "relevance", time = 'all_time', racy = "include",
466                                         author = "", lr = "", categories = "", sortOrder = "ascending",
467                                         callback = None, errorback = None):
468                 print "[MyTube] MyTubePlayerService - search()"
469                 self.feedentries = []
470                 query = gdata.youtube.service.YouTubeVideoQuery()
471                 query.vq = searchTerms
472                 query.orderby = orderby
473                 query.time = time
474                 query.racy = racy
475                 query.sortorder = sortOrder
476                 if lr is not None:
477                         query.lr = lr
478                 if categories[0] is not None:
479                         query.categories = categories
480                 query.start_index = startIndex
481                 query.max_results = maxResults
482                 queryThread = YoutubeQueryThread(self.yt_service.YouTubeQuery, query, self.gotFeed, self.gotFeedError, callback, errorback)
483                 queryThread.start()
484                 return queryThread
485
486         def gotFeed(self, feed, callback):
487                 if feed is not None:
488                         self.feed = feed
489                         for entry in self.feed.entry:
490                                 MyFeedEntry = MyTubeFeedEntry(self, entry)
491                                 self.feedentries.append(MyFeedEntry)
492                 if callback is not None:
493                         callback(self.feed)
494
495         def gotFeedError(self, exception, errorback):
496                 if errorback is not None:
497                         errorback(exception)
498
499         def SubscribeToUser(self, username):
500                 try:
501                         new_subscription = self.yt_service.AddSubscriptionToChannel(username_to_subscribe_to=username)
502         
503                         if isinstance(new_subscription, gdata.youtube.YouTubeSubscriptionEntry):
504                                 print '[MyTube] MyTubePlayerService: New subscription added'
505                                 return _('New subscription added')
506                         
507                         return _('Unknown error')
508                 except gdata.service.RequestError as req:
509                         return str('Error: ' + str(req[0]["body"]))
510                 except Exception as e:
511                         return str('Error: ' + e)
512         
513         def addToFavorites(self, video_id):
514                 try:
515                         video_entry = self.yt_service.GetYouTubeVideoEntry(video_id=video_id)
516                         response = self.yt_service.AddVideoEntryToFavorites(video_entry)
517                         
518                         # The response, if succesfully posted is a YouTubeVideoEntry
519                         if isinstance(response, gdata.youtube.YouTubeVideoEntry):
520                                 print '[MyTube] MyTubePlayerService: Video successfully added to favorites'
521                                 return _('Video successfully added to favorites')       
522         
523                         return _('Unknown error')
524                 except gdata.service.RequestError as req:
525                         return str('Error: ' + str(req[0]["body"]))
526                 except Exception as e:
527                         return str('Error: ' + e)
528         
529         def getTitle(self):
530                 return self.feed.title.text
531
532         def getEntries(self):
533                 return self.feedentries
534
535         def itemCount(self):
536                 return self.feed.items_per_page.text
537
538         def getTotalResults(self):
539                 if self.feed.total_results is None:
540                         return 0
541                                 
542                 return self.feed.total_results.text
543
544         def getNextFeedEntriesURL(self):
545                 for link in self.feed.link:
546                         if link.rel == "next":
547                                 return link.href
548                 return None
549
550         def getCurrentPage(self):
551                 if self.feed.start_index is None:
552                         return 1
553                 
554                 return int(int(self.feed.start_index.text) / int(self.itemCount())) + 1
555
556 class YoutubeQueryThread(Thread):
557         def __init__(self, query, param, gotFeed, gotFeedError, callback, errorback):
558                 Thread.__init__(self)
559                 self.messagePump = ePythonMessagePump()
560                 self.messages = ThreadQueue()
561                 self.gotFeed = gotFeed
562                 self.gotFeedError = gotFeedError
563                 self.callback = callback
564                 self.errorback = errorback
565                 self.query = query
566                 self.param = param
567                 self.canceled = False
568                 self.messagepPump_conn = self.messagePump.recv_msg.connect(self.finished)
569
570         def cancel(self):
571                 self.canceled = True
572
573         def run(self):
574                 try:
575                         if self.param is None:
576                                 feed = self.query()
577                         else:
578                                 feed = self.query(self.param)
579                         self.messages.push((True, feed, self.callback))
580                         self.messagePump.send(0)
581                 except Exception, ex:
582                         self.messages.push((False, ex, self.errorback))
583                         self.messagePump.send(0)
584
585         def finished(self, val):
586                 if not self.canceled:
587                         message = self.messages.pop()
588                         if message[0]:
589                                 self.gotFeed(message[1], message[2])
590                         else:
591                                 self.gotFeedError(message[1], message[2])
592
593 myTubeService = MyTubePlayerService()
594