Merge remote-tracking branch 'remotes/origin/master' into webif_js_rewrite
[enigma2-plugins.git] / autotimer / src / AutoTimer.py
1 from __future__ import print_function
2
3 # Plugins Config
4 from xml.etree.cElementTree import parse as cet_parse
5 from os import path as os_path
6 from AutoTimerConfiguration import parseConfig, buildConfig
7
8 # Navigation (RecordTimer)
9 import NavigationInstance
10
11 # Timer
12 from ServiceReference import ServiceReference
13 from RecordTimer import RecordTimerEntry
14 from Components.TimerSanityCheck import TimerSanityCheck
15
16 # Timespan
17 from time import localtime, strftime, time, mktime
18 from datetime import timedelta, date
19
20 # EPGCache & Event
21 from enigma import eEPGCache, eServiceReference, eServiceCenter, iServiceInformation
22
23 # Enigma2 Config
24 from Components.config import config
25
26 # AutoTimer Component
27 from AutoTimerComponent import preferredAutoTimerComponent
28
29 from itertools import chain
30 from collections import defaultdict
31 from difflib import SequenceMatcher
32 from operator import itemgetter
33
34 from . import xrange, itervalues
35
36 XML_CONFIG = "/etc/enigma2/autotimer.xml"
37
38 def getTimeDiff(timer, begin, end):
39         if begin <= timer.begin <= end:
40                 return end - timer.begin
41         elif timer.begin <= begin <= timer.end:
42                 return timer.end - begin
43         return 0
44
45 typeMap = {
46         "exact": eEPGCache.EXAKT_TITLE_SEARCH,
47         "partial": eEPGCache.PARTIAL_TITLE_SEARCH,
48         "description": -99
49 }
50
51 caseMap = {
52         "sensitive": eEPGCache.CASE_CHECK,
53         "insensitive": eEPGCache.NO_CASE_CHECK
54 }
55
56 class AutoTimer:
57         """Read and save xml configuration, query EPGCache"""
58
59         def __init__(self):
60                 # Initialize
61                 self.timers = []
62                 self.configMtime = -1
63                 self.uniqueTimerId = 0
64                 self.defaultTimer = preferredAutoTimerComponent(
65                         0,              # Id
66                         "",             # Name
67                         "",             # Match
68                         True    # Enabled
69                 )
70
71 # Configuration
72
73         def readXml(self):
74                 # Abort if no config found
75                 if not os_path.exists(XML_CONFIG):
76                         print("[AutoTimer] No configuration file present")
77                         return
78
79                 # Parse if mtime differs from whats saved
80                 mtime = os_path.getmtime(XML_CONFIG)
81                 if mtime == self.configMtime:
82                         print("[AutoTimer] No changes in configuration, won't parse")
83                         return
84
85                 # Save current mtime
86                 self.configMtime = mtime
87
88                 # Parse Config
89                 configuration = cet_parse(XML_CONFIG).getroot()
90
91                 # Empty out timers and reset Ids
92                 del self.timers[:]
93                 self.defaultTimer.clear(-1, True)
94
95                 parseConfig(
96                         configuration,
97                         self.timers,
98                         configuration.get("version"),
99                         0,
100                         self.defaultTimer
101                 )
102                 self.uniqueTimerId = len(self.timers)
103
104         def getXml(self):
105                 return buildConfig(self.defaultTimer, self.timers, webif = True)
106
107         def writeXml(self):
108                 file = open(XML_CONFIG, 'w')
109                 file.writelines(buildConfig(self.defaultTimer, self.timers))
110                 file.close()
111
112 # Manage List
113
114         def add(self, timer):
115                 self.timers.append(timer)
116
117         def getEnabledTimerList(self):
118                 return (x for x in self.timers if x.enabled)
119
120         def getTimerList(self):
121                 return self.timers
122
123         def getTupleTimerList(self):
124                 lst = self.timers
125                 return [(x,) for x in lst]
126
127         def getSortedTupleTimerList(self):
128                 lst = self.timers[:]
129                 lst.sort()
130                 return [(x,) for x in lst]
131
132         def getUniqueId(self):
133                 self.uniqueTimerId += 1
134                 return self.uniqueTimerId
135
136         def remove(self, uniqueId):
137                 idx = 0
138                 for timer in self.timers:
139                         if timer.id == uniqueId:
140                                 self.timers.pop(idx)
141                                 return
142                         idx += 1
143
144         def set(self, timer):
145                 idx = 0
146                 for stimer in self.timers:
147                         if stimer == timer:
148                                 self.timers[idx] = timer
149                                 return
150                         idx += 1
151                 self.timers.append(timer)
152
153 # Main function
154
155         def parseEPG(self, simulateOnly = False):
156                 if NavigationInstance.instance is None:
157                         print("[AutoTimer] Navigation is not available, can't parse EPG")
158                         return (0, 0, 0, [], [], [])
159
160                 total = 0
161                 new = 0
162                 modified = 0
163                 timers = []
164                 conflicting = []
165                 similar = defaultdict(list)                     # Contains the the marked similar eits and the conflicting strings
166                 similars = []                                                                           # Contains the added similar timers
167
168                 # NOTE: the config option specifies "the next X days" which means today (== 1) + X
169                 delta = timedelta(days = config.plugins.autotimer.maxdaysinfuture.value + 1)
170                 evtLimit = mktime((date.today() + delta).timetuple())
171                 checkEvtLimit = delta.days > 1
172                 del delta
173
174                 # Read AutoTimer configuration
175                 self.readXml()
176
177                 # Get E2 instances
178                 epgcache = eEPGCache.getInstance()
179                 serviceHandler = eServiceCenter.getInstance()
180                 recordHandler = NavigationInstance.instance.RecordTimer
181
182                 # Save Recordings in a dict to speed things up a little
183                 # We include processed timers as we might search for duplicate descriptions
184                 # The recordict is always filled
185                 #Question: It might be better to name it timerdict
186                 #Question: Move to a separate function getTimerDict()
187                 #Note: It is also possible to use RecordTimer isInTimer(), but we won't get the timer itself on a match
188                 recorddict = defaultdict(list)
189                 for timer in chain(recordHandler.timer_list, recordHandler.processed_timers):
190                         if timer and timer.service_ref:
191                                 if timer.eit is not None:
192                                         event = epgcache.lookupEventId(timer.service_ref.ref, timer.eit)
193                                         extdesc = event and event.getExtendedDescription() or ''
194                                         timer.extdesc = extdesc
195                                 elif not hasattr(timer, 'extdesc'):
196                                         timer.extdesc = ''
197                                 recorddict[str(timer.service_ref)].append(timer)
198
199                 # Create dict of all movies in all folders used by an autotimer to compare with recordings
200                 # The moviedict will be filled only if one AutoTimer is configured to avoid duplicate description for any recordings
201                 #Question: It might be better to name it recorddict
202                 moviedict = defaultdict(list)
203
204                 # Iterate Timer
205                 for timer in self.getEnabledTimerList():
206                         # Precompute timer destination dir
207                         dest = timer.destination or config.usage.default_path.value
208
209                         # Workaround to allow search for umlauts if we know the encoding
210                         match = timer.match
211                         if timer.encoding != 'UTF-8':
212                                 try:
213                                         match = match.decode('UTF-8').encode(timer.encoding)
214                                 except UnicodeDecodeError:
215                                         pass
216
217                         if timer.searchType == "description":
218                                 test = []
219                                 epgmatches = []
220
221                                 casesensitive = timer.searchCase == "sensitive"
222                                 if not casesensitive:
223                                         match = match.lower()
224
225                                 #if timer.services or timer.bouquets:
226                                 # Service filter defined
227                                 # Search only using the specified services
228                                 for service in timer.services:
229                                         test.append( (service, 0, -1, -1 ) )
230                                 mask = (eServiceReference.isMarker | eServiceReference.isDirectory)
231                                 for bouquet in timer.bouquets:
232                                         services = serviceHandler.list(eServiceReference(bouquet))
233                                         if not services is None:
234                                                 while True:
235                                                         service = services.getNext()
236                                                         if not service.valid(): #check end of list
237                                                                 break
238                                                         if not (service.flags & mask):
239                                                                 test.append( (service.toString(), 0, -1, -1 ) )
240
241                                 if not test:
242                                 #else:
243                                         # No service filter defined
244                                         # Search within all services - could be very slow
245
246                                         # Get all bouquets
247                                         bouquetlist = []
248                                         refstr = '1:134:1:0:0:0:0:0:0:0:FROM BOUQUET \"bouquets.tv\" ORDER BY bouquet'
249                                         bouquetroot = eServiceReference(refstr)
250                                         mask = eServiceReference.isDirectory
251                                         if config.usage.multibouquet.value:
252                                                 bouquets = serviceHandler.list(bouquetroot)
253                                                 if bouquets:
254                                                         while True:
255                                                                 s = bouquets.getNext()
256                                                                 if not s.valid():
257                                                                         break
258                                                                 if s.flags & mask:
259                                                                         info = serviceHandler.info(s)
260                                                                         if info:
261                                                                                 bouquetlist.append((info.getName(s), s))
262                                         else:
263                                                 info = serviceHandler.info(bouquetroot)
264                                                 if info:
265                                                         bouquetlist.append((info.getName(bouquetroot), bouquetroot))
266
267                                         # Get all services
268                                         mask = (eServiceReference.isMarker | eServiceReference.isDirectory)
269                                         for name, bouquet in bouquetlist:
270                                                 if not bouquet.valid(): #check end of list
271                                                         break
272                                                 if bouquet.flags & eServiceReference.isDirectory:
273                                                         services = serviceHandler.list(bouquet)
274                                                         if not services is None:
275                                                                 while True:
276                                                                         service = services.getNext()
277                                                                         if not service.valid(): #check end of list
278                                                                                 break
279                                                                         if not (service.flags & mask):
280                                                                                 test.append( (service.toString(), 0, -1, -1 ) )
281
282                                 if test:
283                                         # Get all events
284                                         #  eEPGCache.lookupEvent( [ format of the returned tuples, ( service, 0 = event intersects given start_time, start_time -1 for now_time), ] )
285                                         test.insert(0, 'RITBDSE')
286                                         allevents = epgcache.lookupEvent( test ) or []
287
288                                         # Filter events
289                                         for serviceref, eit, name, begin, duration, shortdesc, extdesc in allevents:
290                                                 if match in (shortdesc if casesensitive else shortdesc.lower()) \
291                                                         or match in (extdesc if casesensitive else extdesc.lower()):
292                                                         epgmatches.append( (serviceref, eit, name, begin, duration, shortdesc, extdesc) )
293
294                         else:
295                                 # Search EPG, default to empty list
296                                 epgmatches = epgcache.search( ('RITBDSE', 1000, typeMap[timer.searchType], match, caseMap[timer.searchCase]) ) or []
297
298                         # Sort list of tuples by begin time 'B'
299                         epgmatches.sort(key=itemgetter(3))
300
301                         # Reset the the marked similar servicerefs
302                         similar.clear()
303
304                         # Loop over all EPG matches
305                         for idx, ( serviceref, eit, name, begin, duration, shortdesc, extdesc ) in enumerate( epgmatches ):
306
307                                 #Question: Do we need this?
308                                 #Question: Move to separate function getRealService()
309                                 eserviceref = eServiceReference(serviceref)
310                                 evt = epgcache.lookupEventId(eserviceref, eit)
311                                 if not evt:
312                                         print("[AutoTimer] Could not create Event!")
313                                         continue
314                                 # Try to determine real service (we always choose the last one)
315                                 n = evt.getNumOfLinkageServices()
316                                 if n > 0:
317                                         i = evt.getLinkageService(eserviceref, n-1)
318                                         serviceref = i.toString()
319
320                                 evtBegin = begin
321                                 evtEnd = end = begin + duration
322
323                                 # If event starts in less than 60 seconds skip it
324                                 if begin < time() + 60:
325                                         print("[AutoTimer] Skipping an event because it starts in less than 60 seconds")
326                                         continue
327
328                                 # Convert begin time
329                                 timestamp = localtime(begin)
330                                 # Update timer
331                                 timer.update(begin, timestamp)
332
333                                 # Check if eit is in similar matches list
334                                 # NOTE: ignore evtLimit for similar timers as I feel this makes the feature unintuitive
335                                 similarTimer = False
336                                 if eit in similar:
337                                         similarTimer = True
338                                         dayofweek = None # NOTE: ignore day on similar timer
339                                 else:
340                                         # If maximum days in future is set then check time
341                                         if checkEvtLimit:
342                                                 if begin > evtLimit:
343                                                         continue
344
345                                         dayofweek = str(timestamp.tm_wday)
346
347                                 # Check timer conditions
348                                 # NOTE: similar matches to not care about the day/time they are on, so ignore them
349                                 if timer.checkServices(serviceref) \
350                                         or timer.checkDuration(duration) \
351                                         or (not similarTimer and (\
352                                                 timer.checkTimespan(timestamp) \
353                                                 or timer.checkTimeframe(begin) \
354                                         )) or timer.checkFilter(name, shortdesc, extdesc, dayofweek):
355                                         continue
356
357                                 if timer.hasOffset():
358                                         # Apply custom Offset
359                                         begin, end = timer.applyOffset(begin, end)
360                                 else:
361                                         # Apply E2 Offset
362                                         begin -= config.recording.margin_before.value * 60
363                                         end += config.recording.margin_after.value * 60
364
365                                 # Eventually change service to alternative
366                                 if timer.overrideAlternatives:
367                                         serviceref = timer.getAlternative(serviceref)
368
369                                 total += 1
370
371                                 # Append to timerlist and abort if simulating
372                                 timers.append((name, begin, end, serviceref, timer.name))
373                                 if simulateOnly:
374                                         continue
375
376                                 # Check for existing recordings in directory
377                                 if timer.avoidDuplicateDescription == 3:
378                                         # Reset movie Exists
379                                         movieExists = False
380
381                                         # Eventually create cache
382                                         if dest and dest not in moviedict:
383                                                 #Question: Move to a separate function getRecordDict()
384                                                 movielist = serviceHandler.list(eServiceReference("2:0:1:0:0:0:0:0:0:0:" + dest))
385                                                 if movielist is None:
386                                                         print("[AutoTimer] listing of movies in " + dest + " failed")
387                                                 else:
388                                                         append = moviedict[dest].append
389                                                         while 1:
390                                                                 movieref = movielist.getNext()
391                                                                 if not movieref.valid():
392                                                                         break
393                                                                 if movieref.flags & eServiceReference.mustDescent:
394                                                                         continue
395                                                                 info = serviceHandler.info(movieref)
396                                                                 if info is None:
397                                                                         continue
398                                                                 event = info.getEvent(movieref)
399                                                                 if event is None:
400                                                                         continue
401                                                                 append({
402                                                                         "name": info.getName(movieref),
403                                                                         "shortdesc": info.getInfoString(movieref, iServiceInformation.sDescription),
404                                                                         "extdesc": event.getExtendedDescription() or '' # XXX: does event.getExtendedDescription() actually return None on no description or an empty string?
405                                                                 })
406                                                         del append
407
408                                         for movieinfo in moviedict.get(dest, ()):
409                                                 if movieinfo.get("name") == name \
410                                                         and movieinfo.get("shortdesc") == shortdesc:
411                                                         # Some channels indicate replays in the extended descriptions
412                                                         # If the similarity percent is higher then 0.8 it is a very close match
413                                                         extdescM = movieinfo.get("extdesc")
414                                                         if ( len(extdesc) == len(extdescM) and extdesc == extdescM ) \
415                                                                 or ( 0.8 < SequenceMatcher(lambda x: x == " ",extdesc, extdescM).ratio() ):
416                                                                 print("[AutoTimer] We found a matching recorded movie, skipping event:", name)
417                                                                 movieExists = True
418                                                                 break
419
420                                         if movieExists:
421                                                 continue
422
423                                 # Initialize
424                                 newEntry = None
425                                 oldExists = False
426
427                                 # Check for double Timers
428                                 # We first check eit and if user wants us to guess event based on time
429                                 # we try this as backup. The allowed diff should be configurable though.
430                                 for rtimer in recorddict.get(serviceref, ()):
431                                         if rtimer.eit == eit or config.plugins.autotimer.try_guessing.value and getTimeDiff(rtimer, evtBegin, evtEnd) > ((duration/10)*8):
432                                                 oldExists = True
433
434                                                 # Abort if we don't want to modify timers or timer is repeated
435                                                 if config.plugins.autotimer.refresh.value == "none" or rtimer.repeated:
436                                                         print("[AutoTimer] Won't modify existing timer because either no modification allowed or repeated timer")
437                                                         break
438
439                                                 if hasattr(rtimer, "isAutoTimer"):
440                                                         rtimer.log(501, "[AutoTimer] AutoTimer %s modified this automatically generated timer." % (timer.name))
441                                                 else:
442                                                         if config.plugins.autotimer.refresh.value != "all":
443                                                                 print("[AutoTimer] Won't modify existing timer because it's no timer set by us")
444                                                                 break
445
446                                                         rtimer.log(501, "[AutoTimer] Warning, AutoTimer %s messed with a timer which might not belong to it." % (timer.name))
447
448                                                 newEntry = rtimer
449                                                 modified += 1
450
451                                                 # Modify values saved in timer
452                                                 newEntry.name = name
453                                                 newEntry.description = shortdesc
454                                                 newEntry.begin = int(begin)
455                                                 newEntry.end = int(end)
456                                                 newEntry.service_ref = ServiceReference(serviceref)
457
458                                                 break
459                                         elif timer.avoidDuplicateDescription >= 1 \
460                                                 and not rtimer.disabled \
461                                                 and rtimer.name == name \
462                                                 and rtimer.description == shortdesc:
463                                                         # Some channels indicate replays in the extended descriptions
464                                                         # If the similarity percent is higher then 0.8 it is a very close match
465                                                         if ( len(extdesc) == len(rtimer.extdesc) and extdesc == rtimer.extdesc ) \
466                                                                 or ( 0.8 < SequenceMatcher(lambda x: x == " ",extdesc, rtimer.extdesc).ratio() ):
467                                                                 oldExists = True
468                                                                 print("[AutoTimer] We found a timer (similar service) with same description, skipping event")
469                                                                 break
470
471                                 # We found no timer we want to edit
472                                 if newEntry is None:
473                                         # But there is a match
474                                         if oldExists:
475                                                 continue
476
477                                         # We want to search for possible doubles
478                                         if timer.avoidDuplicateDescription >= 2:
479                                                 for rtimer in chain.from_iterable( itervalues(recorddict) ):
480                                                         if not rtimer.disabled \
481                                                                 and rtimer.name == name \
482                                                                 and rtimer.description == shortdesc:
483                                                                         # Some channels indicate replays in the extended descriptions
484                                                                         # If the similarity percent is higher then 0.8 it is a very close match
485                                                                         if ( len(extdesc) == len(rtimer.extdesc) and extdesc == rtimer.extdesc ) \
486                                                                                 or ( 0.8 < SequenceMatcher(lambda x: x == " ",extdesc, rtimer.extdesc).ratio() ):
487                                                                                 oldExists = True
488                                                                                 print("[AutoTimer] We found a timer (any service) with same description, skipping event")
489                                                                                 break
490                                                 if oldExists:
491                                                         continue
492
493                                         if timer.checkCounter(timestamp):
494                                                 print("[AutoTimer] Not adding new timer because counter is depleted.")
495                                                 continue
496
497                                         newEntry = RecordTimerEntry(ServiceReference(serviceref), begin, end, name, shortdesc, eit)
498                                         newEntry.log(500, "[AutoTimer] Try to add new timer based on AutoTimer %s." % (timer.name))
499
500                                         # Mark this entry as AutoTimer (only AutoTimers will have this Attribute set)
501                                         # It is only temporarily, after a restart it will be lost,
502                                         # because it won't be stored in the timer xml file
503                                         newEntry.isAutoTimer = True
504
505                                 # Apply afterEvent
506                                 if timer.hasAfterEvent():
507                                         afterEvent = timer.getAfterEventTimespan(localtime(end))
508                                         if afterEvent is None:
509                                                 afterEvent = timer.getAfterEvent()
510                                         if afterEvent is not None:
511                                                 newEntry.afterEvent = afterEvent
512
513                                 newEntry.dirname = timer.destination
514                                 newEntry.justplay = timer.justplay
515                                 newEntry.tags = timer.tags
516                                 newEntry.vpsplugin_enabled = timer.vps_enabled
517                                 newEntry.vpsplugin_overwrite = timer.vps_overwrite
518
519                                 if oldExists:
520                                         # XXX: this won't perform a sanity check, but do we actually want to do so?
521                                         recordHandler.timeChanged(newEntry)
522                                 else:
523                                         conflictString = ""
524                                         if similarTimer:
525                                                 conflictString = similar[eit].conflictString
526                                                 newEntry.log(504, "[AutoTimer] Try to add similar Timer because of conflicts with %s." % (conflictString))
527
528                                         # Try to add timer
529                                         conflicts = recordHandler.record(newEntry)
530
531                                         if conflicts:
532                                                 conflictString += ' / '.join(["%s (%s)" % (x.name, strftime("%Y%m%d %H%M", localtime(x.begin))) for x in conflicts])
533                                                 print("[AutoTimer] conflict with %s detected" % (conflictString))
534
535                                         if conflicts and config.plugins.autotimer.addsimilar_on_conflict.value:
536                                                 # We start our search right after our actual index
537                                                 # Attention we have to use a copy of the list, because we have to append the previous older matches
538                                                 lepgm = len(epgmatches)
539                                                 for i in xrange(lepgm):
540                                                         servicerefS, eitS, nameS, beginS, durationS, shortdescS, extdescS = epgmatches[ (i+idx+1)%lepgm ]
541                                                         if shortdesc == shortdescS:
542                                                                 # Some channels indicate replays in the extended descriptions
543                                                                 # If the similarity percent is higher then 0.8 it is a very close match
544                                                                 if ( len(extdesc) == len(extdescS) and extdesc == extdescS ) \
545                                                                         or ( 0.8 < SequenceMatcher(lambda x: x == " ",extdesc, extdescS).ratio() ):
546                                                                         # Check if the similar is already known
547                                                                         if eitS not in similar:
548                                                                                 print("[AutoTimer] Found similar Timer: " + name)
549
550                                                                                 # Store the actual and similar eit and conflictString, so it can be handled later
551                                                                                 newEntry.conflictString = conflictString
552                                                                                 similar[eit] = newEntry
553                                                                                 similar[eitS] = newEntry
554                                                                                 similarTimer = True
555                                                                                 if beginS <= evtBegin:
556                                                                                         # Event is before our actual epgmatch so we have to append it to the epgmatches list
557                                                                                         epgmatches.append((servicerefS, eitS, nameS, beginS, durationS, shortdescS, extdescS))
558                                                                                 # If we need a second similar it will be found the next time
559                                                                                 break
560                                                                         else:
561                                                                                 similarTimer = False
562                                                                                 newEntry = similar[eitS]
563                                                                                 break
564
565                                         if conflicts is None:
566                                                 timer.decrementCounter()
567                                                 new += 1
568                                                 newEntry.extdesc = extdesc
569                                                 recorddict[serviceref].append(newEntry)
570
571                                                 # Similar timers are in new timers list and additionally in similar timers list
572                                                 if similarTimer:
573                                                         similars.append((name, begin, end, serviceref, timer.name))
574                                                         similar.clear()
575
576                                         # Don't care about similar timers
577                                         elif not similarTimer:
578                                                 conflicting.append((name, begin, end, serviceref, timer.name))
579
580                                                 if config.plugins.autotimer.disabled_on_conflict.value:
581                                                         newEntry.log(503, "[AutoTimer] Timer disabled because of conflicts with %s." % (conflictString))
582                                                         newEntry.disabled = True
583                                                         # We might want to do the sanity check locally so we don't run it twice - but I consider this workaround a hack anyway
584                                                         conflicts = recordHandler.record(newEntry)
585
586                 return (total, new, modified, timers, conflicting, similars)