autotimer: more control over duplicate search
[enigma2-plugins.git] / autotimer / src / AutoTimer.py
1 from __future__ import print_function
2
3 # Plugins Config
4 from xml.etree.cElementTree import parse as cet_parse
5 from os import path as os_path
6 from AutoTimerConfiguration import parseConfig, buildConfig
7
8 # Navigation (RecordTimer)
9 import NavigationInstance
10
11 # Timer
12 from ServiceReference import ServiceReference
13 from RecordTimer import RecordTimerEntry
14 from Components.TimerSanityCheck import TimerSanityCheck
15
16 # Timespan
17 from time import localtime, strftime, time, mktime
18 from datetime import timedelta, date
19
20 # EPGCache & Event
21 from enigma import eEPGCache, eServiceReference, eServiceCenter, iServiceInformation
22
23 # Enigma2 Config
24 from Components.config import config
25
26 # AutoTimer Component
27 from AutoTimerComponent import preferredAutoTimerComponent
28
29 from itertools import chain
30 from collections import defaultdict
31 from difflib import SequenceMatcher
32 from operator import itemgetter
33
34 from . import xrange, itervalues
35
36 XML_CONFIG = "/etc/enigma2/autotimer.xml"
37
38 def getTimeDiff(timer, begin, end):
39         if begin <= timer.begin <= end:
40                 return end - timer.begin
41         elif timer.begin <= begin <= timer.end:
42                 return timer.end - begin
43         return 0
44
45 typeMap = {
46         "exact": eEPGCache.EXAKT_TITLE_SEARCH,
47         "partial": eEPGCache.PARTIAL_TITLE_SEARCH,
48         "description": -99
49 }
50
51 caseMap = {
52         "sensitive": eEPGCache.CASE_CHECK,
53         "insensitive": eEPGCache.NO_CASE_CHECK
54 }
55
56 class AutoTimer:
57         """Read and save xml configuration, query EPGCache"""
58
59         def __init__(self):
60                 # Initialize
61                 self.timers = []
62                 self.configMtime = -1
63                 self.uniqueTimerId = 0
64                 self.defaultTimer = preferredAutoTimerComponent(
65                         0,              # Id
66                         "",             # Name
67                         "",             # Match
68                         True    # Enabled
69                 )
70
71 # Configuration
72
73         def readXml(self):
74                 # Abort if no config found
75                 if not os_path.exists(XML_CONFIG):
76                         print("[AutoTimer] No configuration file present")
77                         return
78
79                 # Parse if mtime differs from whats saved
80                 mtime = os_path.getmtime(XML_CONFIG)
81                 if mtime == self.configMtime:
82                         print("[AutoTimer] No changes in configuration, won't parse")
83                         return
84
85                 # Save current mtime
86                 self.configMtime = mtime
87
88                 # Parse Config
89                 configuration = cet_parse(XML_CONFIG).getroot()
90
91                 # Empty out timers and reset Ids
92                 del self.timers[:]
93                 self.defaultTimer.clear(-1, True)
94
95                 parseConfig(
96                         configuration,
97                         self.timers,
98                         configuration.get("version"),
99                         0,
100                         self.defaultTimer
101                 )
102                 self.uniqueTimerId = len(self.timers)
103
104         def getXml(self):
105                 return buildConfig(self.defaultTimer, self.timers, webif = True)
106
107         def writeXml(self):
108                 file = open(XML_CONFIG, 'w')
109                 file.writelines(buildConfig(self.defaultTimer, self.timers))
110                 file.close()
111
112 # Manage List
113
114         def add(self, timer):
115                 self.timers.append(timer)
116
117         def getEnabledTimerList(self):
118                 return (x for x in self.timers if x.enabled)
119
120         def getTimerList(self):
121                 return self.timers
122
123         def getTupleTimerList(self):
124                 lst = self.timers
125                 return [(x,) for x in lst]
126
127         def getSortedTupleTimerList(self):
128                 lst = self.timers[:]
129                 lst.sort()
130                 return [(x,) for x in lst]
131
132         def getUniqueId(self):
133                 self.uniqueTimerId += 1
134                 return self.uniqueTimerId
135
136         def remove(self, uniqueId):
137                 idx = 0
138                 for timer in self.timers:
139                         if timer.id == uniqueId:
140                                 self.timers.pop(idx)
141                                 return
142                         idx += 1
143
144         def set(self, timer):
145                 idx = 0
146                 for stimer in self.timers:
147                         if stimer == timer:
148                                 self.timers[idx] = timer
149                                 return
150                         idx += 1
151                 self.timers.append(timer)
152
153 # Main function
154
155         def parseEPG(self, simulateOnly = False):
156                 if NavigationInstance.instance is None:
157                         print("[AutoTimer] Navigation is not available, can't parse EPG")
158                         return (0, 0, 0, [], [], [])
159
160                 total = 0
161                 new = 0
162                 modified = 0
163                 timers = []
164                 conflicting = []
165                 similar = defaultdict(list)                     # Contains the the marked similar eits and the conflicting strings
166                 similars = []                                                                           # Contains the added similar timers
167
168                 # NOTE: the config option specifies "the next X days" which means today (== 1) + X
169                 delta = timedelta(days = config.plugins.autotimer.maxdaysinfuture.value + 1)
170                 evtLimit = mktime((date.today() + delta).timetuple())
171                 checkEvtLimit = delta.days > 1
172                 del delta
173
174                 # Read AutoTimer configuration
175                 self.readXml()
176
177                 # Get E2 instances
178                 epgcache = eEPGCache.getInstance()
179                 serviceHandler = eServiceCenter.getInstance()
180                 recordHandler = NavigationInstance.instance.RecordTimer
181
182                 # Save Recordings in a dict to speed things up a little
183                 # We include processed timers as we might search for duplicate descriptions
184                 # The recordict is always filled
185                 #Question: It might be better to name it timerdict
186                 #Question: Move to a separate function getTimerDict()
187                 #Note: It is also possible to use RecordTimer isInTimer(), but we won't get the timer itself on a match
188                 recorddict = defaultdict(list)
189                 for timer in chain(recordHandler.timer_list, recordHandler.processed_timers):
190                         if timer and timer.service_ref:
191                                 if timer.eit is not None:
192                                         event = epgcache.lookupEventId(timer.service_ref.ref, timer.eit)
193                                         extdesc = event and event.getExtendedDescription() or ''
194                                         timer.extdesc = extdesc
195                                 elif not hasattr(timer, 'extdesc'):
196                                         timer.extdesc = ''
197                                 recorddict[str(timer.service_ref)].append(timer)
198
199                 # Create dict of all movies in all folders used by an autotimer to compare with recordings
200                 # The moviedict will be filled only if one AutoTimer is configured to avoid duplicate description for any recordings
201                 #Question: It might be better to name it recorddict
202                 moviedict = defaultdict(list)
203
204                 # Iterate Timer
205                 for timer in self.getEnabledTimerList():
206                         # Precompute timer destination dir
207                         dest = timer.destination or config.usage.default_path.value
208
209                         # Workaround to allow search for umlauts if we know the encoding
210                         match = timer.match
211                         if timer.encoding != 'UTF-8':
212                                 try:
213                                         match = match.decode('UTF-8').encode(timer.encoding)
214                                 except UnicodeDecodeError:
215                                         pass
216
217                         if timer.searchType == "description":
218                                 test = []
219                                 epgmatches = []
220
221                                 casesensitive = timer.searchCase == "sensitive"
222                                 if not casesensitive:
223                                         match = match.lower()
224
225                                 #if timer.services or timer.bouquets:
226                                 # Service filter defined
227                                 # Search only using the specified services
228                                 for service in timer.services:
229                                         test.append( (service, 0, -1, -1 ) )
230                                 mask = (eServiceReference.isMarker | eServiceReference.isDirectory)
231                                 for bouquet in timer.bouquets:
232                                         services = serviceHandler.list(eServiceReference(bouquet))
233                                         if not services is None:
234                                                 while True:
235                                                         service = services.getNext()
236                                                         if not service.valid(): #check end of list
237                                                                 break
238                                                         if not (service.flags & mask):
239                                                                 test.append( (service.toString(), 0, -1, -1 ) )
240
241                                 if not test:
242                                 #else:
243                                         # No service filter defined
244                                         # Search within all services - could be very slow
245
246                                         # Get all bouquets
247                                         bouquetlist = []
248                                         refstr = '1:134:1:0:0:0:0:0:0:0:FROM BOUQUET \"bouquets.tv\" ORDER BY bouquet'
249                                         bouquetroot = eServiceReference(refstr)
250                                         mask = eServiceReference.isDirectory
251                                         if config.usage.multibouquet.value:
252                                                 bouquets = serviceHandler.list(bouquetroot)
253                                                 if bouquets:
254                                                         while True:
255                                                                 s = bouquets.getNext()
256                                                                 if not s.valid():
257                                                                         break
258                                                                 if s.flags & mask:
259                                                                         info = serviceHandler.info(s)
260                                                                         if info:
261                                                                                 bouquetlist.append((info.getName(s), s))
262                                         else:
263                                                 info = serviceHandler.info(bouquetroot)
264                                                 if info:
265                                                         bouquetlist.append((info.getName(bouquetroot), bouquetroot))
266
267                                         # Get all services
268                                         mask = (eServiceReference.isMarker | eServiceReference.isDirectory)
269                                         for name, bouquet in bouquetlist:
270                                                 if not bouquet.valid(): #check end of list
271                                                         break
272                                                 if bouquet.flags & eServiceReference.isDirectory:
273                                                         services = serviceHandler.list(bouquet)
274                                                         if not services is None:
275                                                                 while True:
276                                                                         service = services.getNext()
277                                                                         if not service.valid(): #check end of list
278                                                                                 break
279                                                                         if not (service.flags & mask):
280                                                                                 test.append( (service.toString(), 0, -1, -1 ) )
281
282                                 if test:
283                                         # Get all events
284                                         #  eEPGCache.lookupEvent( [ format of the returned tuples, ( service, 0 = event intersects given start_time, start_time -1 for now_time), ] )
285                                         test.insert(0, 'RITBDSE')
286                                         allevents = epgcache.lookupEvent( test ) or []
287
288                                         # Filter events
289                                         for serviceref, eit, name, begin, duration, shortdesc, extdesc in allevents:
290                                                 if match in (shortdesc if casesensitive else shortdesc.lower()) \
291                                                         or match in (extdesc if casesensitive else extdesc.lower()):
292                                                         epgmatches.append( (serviceref, eit, name, begin, duration, shortdesc, extdesc) )
293
294                         else:
295                                 # Search EPG, default to empty list
296                                 epgmatches = epgcache.search( ('RITBDSE', 1000, typeMap[timer.searchType], match, caseMap[timer.searchCase]) ) or []
297
298                         # Sort list of tuples by begin time 'B'
299                         epgmatches.sort(key=itemgetter(3))
300
301                         # Reset the the marked similar servicerefs
302                         similar.clear()
303
304                         # Loop over all EPG matches
305                         for idx, ( serviceref, eit, name, begin, duration, shortdesc, extdesc ) in enumerate( epgmatches ):
306
307                                 #Question: Do we need this?
308                                 #Question: Move to separate function getRealService()
309                                 eserviceref = eServiceReference(serviceref)
310                                 evt = epgcache.lookupEventId(eserviceref, eit)
311                                 if not evt:
312                                         print("[AutoTimer] Could not create Event!")
313                                         continue
314                                 # Try to determine real service (we always choose the last one)
315                                 n = evt.getNumOfLinkageServices()
316                                 if n > 0:
317                                         i = evt.getLinkageService(eserviceref, n-1)
318                                         serviceref = i.toString()
319
320                                 evtBegin = begin
321                                 evtEnd = end = begin + duration
322
323                                 # If event starts in less than 60 seconds skip it
324                                 if begin < time() + 60:
325                                         print("[AutoTimer] Skipping an event because it starts in less than 60 seconds")
326                                         continue
327
328                                 # Convert begin time
329                                 timestamp = localtime(begin)
330                                 # Update timer
331                                 timer.update(begin, timestamp)
332
333                                 # Check if eit is in similar matches list
334                                 # NOTE: ignore evtLimit for similar timers as I feel this makes the feature unintuitive
335                                 similarTimer = False
336                                 if eit in similar:
337                                         similarTimer = True
338                                         dayofweek = None # NOTE: ignore day on similar timer
339                                 else:
340                                         # If maximum days in future is set then check time
341                                         if checkEvtLimit:
342                                                 if begin > evtLimit:
343                                                         continue
344
345                                         dayofweek = str(timestamp.tm_wday)
346
347                                 # Check timer conditions
348                                 # NOTE: similar matches to not care about the day/time they are on, so ignore them
349                                 if timer.checkServices(serviceref) \
350                                         or timer.checkDuration(duration) \
351                                         or (not similarTimer and (\
352                                                 timer.checkTimespan(timestamp) \
353                                                 or timer.checkTimeframe(begin) \
354                                         )) or timer.checkFilter(name, shortdesc, extdesc, dayofweek):
355                                         continue
356
357                                 if timer.hasOffset():
358                                         # Apply custom Offset
359                                         begin, end = timer.applyOffset(begin, end)
360                                 else:
361                                         # Apply E2 Offset
362                                         begin -= config.recording.margin_before.value * 60
363                                         end += config.recording.margin_after.value * 60
364
365                                 # Eventually change service to alternative
366                                 if timer.overrideAlternatives:
367                                         serviceref = timer.getAlternative(serviceref)
368
369                                 total += 1
370
371                                 # Append to timerlist and abort if simulating
372                                 timers.append((name, begin, end, serviceref, timer.name))
373                                 if simulateOnly:
374                                         continue
375
376                                 # Check for existing recordings in directory
377                                 if timer.avoidDuplicateDescription == 3:
378                                         # Reset movie Exists
379                                         movieExists = False
380
381                                         # Eventually create cache
382                                         if dest and dest not in moviedict:
383                                                 #Question: Move to a separate function getRecordDict()
384                                                 movielist = serviceHandler.list(eServiceReference("2:0:1:0:0:0:0:0:0:0:" + dest))
385                                                 if movielist is None:
386                                                         print("[AutoTimer] listing of movies in " + dest + " failed")
387                                                 else:
388                                                         append = moviedict[dest].append
389                                                         while 1:
390                                                                 movieref = movielist.getNext()
391                                                                 if not movieref.valid():
392                                                                         break
393                                                                 if movieref.flags & eServiceReference.mustDescent:
394                                                                         continue
395                                                                 info = serviceHandler.info(movieref)
396                                                                 if info is None:
397                                                                         continue
398                                                                 event = info.getEvent(movieref)
399                                                                 if event is None:
400                                                                         continue
401                                                                 append({
402                                                                         "name": info.getName(movieref),
403                                                                         "shortdesc": info.getInfoString(movieref, iServiceInformation.sDescription),
404                                                                         "extdesc": event.getExtendedDescription() or '' # XXX: does event.getExtendedDescription() actually return None on no description or an empty string?
405                                                                 })
406                                                         del append
407
408                                         for movieinfo in moviedict.get(dest, ()):
409                                                 if self.checkSimilarity(timer, name, movieinfo.get("name"), shortdesc, movieinfo.get("shortdesc"), extdesc, movieinfo.get("extdesc") ):
410                                                         print("[AutoTimer] We found a matching recorded movie, skipping event:", name)
411                                                         movieExists = True
412                                                         break
413                                         if movieExists:
414                                                 continue
415
416                                 # Initialize
417                                 newEntry = None
418                                 oldExists = False
419
420                                 # Check for double Timers
421                                 # We first check eit and if user wants us to guess event based on time
422                                 # we try this as backup. The allowed diff should be configurable though.
423                                 for rtimer in recorddict.get(serviceref, ()):
424                                         if rtimer.eit == eit or config.plugins.autotimer.try_guessing.value and getTimeDiff(rtimer, evtBegin, evtEnd) > ((duration/10)*8):
425                                                 oldExists = True
426
427                                                 # Abort if we don't want to modify timers or timer is repeated
428                                                 if config.plugins.autotimer.refresh.value == "none" or rtimer.repeated:
429                                                         print("[AutoTimer] Won't modify existing timer because either no modification allowed or repeated timer")
430                                                         break
431
432                                                 if hasattr(rtimer, "isAutoTimer"):
433                                                         rtimer.log(501, "[AutoTimer] AutoTimer %s modified this automatically generated timer." % (timer.name))
434                                                 else:
435                                                         if config.plugins.autotimer.refresh.value != "all":
436                                                                 print("[AutoTimer] Won't modify existing timer because it's no timer set by us")
437                                                                 break
438
439                                                         rtimer.log(501, "[AutoTimer] Warning, AutoTimer %s messed with a timer which might not belong to it." % (timer.name))
440
441                                                 newEntry = rtimer
442                                                 modified += 1
443
444                                                 # Modify values saved in timer
445                                                 newEntry.name = name
446                                                 newEntry.description = shortdesc
447                                                 newEntry.begin = int(begin)
448                                                 newEntry.end = int(end)
449                                                 newEntry.service_ref = ServiceReference(serviceref)
450
451                                                 break
452                                         elif timer.avoidDuplicateDescription >= 1 \
453                                                 and not rtimer.disabled:
454                                                         if self.checkSimilarity(timer, name, rtimer.name, shortdesc, rtimer.description, extdesc, rtimer.extdesc ):
455                                                         # if searchForDuplicateDescription > 1 then check short description
456                                                                 oldExists = True
457                                                                 print("[AutoTimer] We found a timer (similar service) with same description, skipping event")
458                                                                 break
459
460                                 # We found no timer we want to edit
461                                 if newEntry is None:
462                                         # But there is a match
463                                         if oldExists:
464                                                 continue
465
466                                         # We want to search for possible doubles
467                                         if timer.avoidDuplicateDescription >= 2:
468                                                 for rtimer in chain.from_iterable( itervalues(recorddict) ):
469                                                         if not rtimer.disabled:
470                                                                 if self.checkSimilarity(timer, name, rtimer.name, shortdesc, rtimer.description, extdesc, rtimer.extdesc ):
471                                                                         oldExists = True
472                                                                         print("[AutoTimer] We found a timer (any service) with same description, skipping event")
473                                                                         break
474                                                 if oldExists:
475                                                         continue
476
477                                         if timer.checkCounter(timestamp):
478                                                 print("[AutoTimer] Not adding new timer because counter is depleted.")
479                                                 continue
480
481                                         newEntry = RecordTimerEntry(ServiceReference(serviceref), begin, end, name, shortdesc, eit)
482                                         newEntry.log(500, "[AutoTimer] Try to add new timer based on AutoTimer %s." % (timer.name))
483
484                                         # Mark this entry as AutoTimer (only AutoTimers will have this Attribute set)
485                                         # It is only temporarily, after a restart it will be lost,
486                                         # because it won't be stored in the timer xml file
487                                         newEntry.isAutoTimer = True
488
489                                 # Apply afterEvent
490                                 if timer.hasAfterEvent():
491                                         afterEvent = timer.getAfterEventTimespan(localtime(end))
492                                         if afterEvent is None:
493                                                 afterEvent = timer.getAfterEvent()
494                                         if afterEvent is not None:
495                                                 newEntry.afterEvent = afterEvent
496
497                                 newEntry.dirname = timer.destination
498                                 newEntry.justplay = timer.justplay
499                                 newEntry.tags = timer.tags
500                                 newEntry.vpsplugin_enabled = timer.vps_enabled
501                                 newEntry.vpsplugin_overwrite = timer.vps_overwrite
502
503                                 if oldExists:
504                                         # XXX: this won't perform a sanity check, but do we actually want to do so?
505                                         recordHandler.timeChanged(newEntry)
506                                 else:
507                                         conflictString = ""
508                                         if similarTimer:
509                                                 conflictString = similar[eit].conflictString
510                                                 newEntry.log(504, "[AutoTimer] Try to add similar Timer because of conflicts with %s." % (conflictString))
511
512                                         # Try to add timer
513                                         conflicts = recordHandler.record(newEntry)
514
515                                         if conflicts:
516                                                 conflictString += ' / '.join(["%s (%s)" % (x.name, strftime("%Y%m%d %H%M", localtime(x.begin))) for x in conflicts])
517                                                 print("[AutoTimer] conflict with %s detected" % (conflictString))
518
519                                         if conflicts and config.plugins.autotimer.addsimilar_on_conflict.value:
520                                                 # We start our search right after our actual index
521                                                 # Attention we have to use a copy of the list, because we have to append the previous older matches
522                                                 lepgm = len(epgmatches)
523                                                 for i in xrange(lepgm):
524                                                         servicerefS, eitS, nameS, beginS, durationS, shortdescS, extdescS = epgmatches[ (i+idx+1)%lepgm ]
525                                                         if shortdesc == shortdescS:
526                                                                 # Some channels indicate replays in the extended descriptions
527                                                                 # If the similarity percent is higher then 0.8 it is a very close match
528                                                                 if ( len(extdesc) == len(extdescS) and extdesc == extdescS ) \
529                                                                         or ( 0.8 < SequenceMatcher(lambda x: x == " ",extdesc, extdescS).ratio() ):
530                                                                         # Check if the similar is already known
531                                                                         if eitS not in similar:
532                                                                                 print("[AutoTimer] Found similar Timer: " + name)
533
534                                                                                 # Store the actual and similar eit and conflictString, so it can be handled later
535                                                                                 newEntry.conflictString = conflictString
536                                                                                 similar[eit] = newEntry
537                                                                                 similar[eitS] = newEntry
538                                                                                 similarTimer = True
539                                                                                 if beginS <= evtBegin:
540                                                                                         # Event is before our actual epgmatch so we have to append it to the epgmatches list
541                                                                                         epgmatches.append((servicerefS, eitS, nameS, beginS, durationS, shortdescS, extdescS))
542                                                                                 # If we need a second similar it will be found the next time
543                                                                                 break
544                                                                         else:
545                                                                                 similarTimer = False
546                                                                                 newEntry = similar[eitS]
547                                                                                 break
548
549                                         if conflicts is None:
550                                                 timer.decrementCounter()
551                                                 new += 1
552                                                 newEntry.extdesc = extdesc
553                                                 recorddict[serviceref].append(newEntry)
554
555                                                 # Similar timers are in new timers list and additionally in similar timers list
556                                                 if similarTimer:
557                                                         similars.append((name, begin, end, serviceref, timer.name))
558                                                         similar.clear()
559
560                                         # Don't care about similar timers
561                                         elif not similarTimer:
562                                                 conflicting.append((name, begin, end, serviceref, timer.name))
563
564                                                 if config.plugins.autotimer.disabled_on_conflict.value:
565                                                         newEntry.log(503, "[AutoTimer] Timer disabled because of conflicts with %s." % (conflictString))
566                                                         newEntry.disabled = True
567                                                         # We might want to do the sanity check locally so we don't run it twice - but I consider this workaround a hack anyway
568                                                         conflicts = recordHandler.record(newEntry)
569
570                 return (total, new, modified, timers, conflicting, similars)
571
572         def checkSimilarity(self, timer, name1, name2, shortdesc1, shortdesc2, extdesc1, extdesc2 ):
573                 # Always check title
574                 foundTitle = (name1 == name2)
575                 print("[AutoTimer] Check name:" + name1 + "=" + name2 + ">>", foundTitle)
576                 if timer.searchForDuplicateDescription >= 2: # Check titles if searchForDuplicateDescription > 2
577                         foundShort = (shortdesc1 == shortdesc2)
578                         print("[AutoTimer] Check short desc:" + shortdesc1 + "=" + shortdesc2 + ">>", foundShort)
579                 else:
580                         foundShort = True;
581                 if timer.searchForDuplicateDescription == 3: # Check Extended if searchForDuplicateDescription = 3
582                         # Some channels indicate replays in the extended descriptions
583                         # If the similarity percent is higher then 0.8 it is a very close match
584                         foundExt = ( len(extdesc1) == len(extdesc2) and extdesc1 == extdesc2 ) \
585                          or ( 0.8 < SequenceMatcher(lambda x: x == " ",extdesc1, extdesc2).ratio() )
586                         print("[AutoTimer] Check ext desc:" + extdesc1[:10] + "=" + extdesc2[:10] + ">>", foundExt)
587                 else:
588                         foundExt = True;
589                 retVal = (foundTitle and foundShort and foundExt)
590                 print("[AutoTimer] Check similarity returning >>", retVal)
591
592                 return retVal