|
1
|
# zap2epg tv schedule grabber for kodi
|
|
2
|
################################################################################
|
|
3
|
# This program is free software: you can redistribute it and/or modify
|
|
4
|
# it under the terms of the GNU General Public License as published by
|
|
5
|
# the Free Software Foundation, either version 3 of the License, or
|
|
6
|
# (at your option) any later version.
|
|
7
|
#
|
|
8
|
# This program is distributed in the hope that it will be useful,
|
|
9
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11
|
# GNU General Public License for more details.
|
|
12
|
#
|
|
13
|
# You should have received a copy of the GNU General Public License
|
|
14
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
15
|
################################################################################
|
|
16
|
|
|
17
|
import urllib2
|
|
18
|
import base64
|
|
19
|
import codecs
|
|
20
|
import time
|
|
21
|
import datetime
|
|
22
|
import _strptime
|
|
23
|
import calendar
|
|
24
|
import gzip
|
|
25
|
import os
|
|
26
|
import logging
|
|
27
|
import re
|
|
28
|
import json
|
|
29
|
import sys
|
|
30
|
from os.path import dirname
|
|
31
|
import xml.etree.ElementTree as ET
|
|
32
|
from collections import OrderedDict
|
|
33
|
import hashlib
|
|
34
|
|
|
35
|
|
|
36
|
def mainRun(userdata):
|
|
37
|
settingsFile = os.path.join(userdata, 'settings.xml')
|
|
38
|
settings = ET.parse(settingsFile)
|
|
39
|
root = settings.getroot()
|
|
40
|
settingsDict = {}
|
|
41
|
xdescOrderDict = {}
|
|
42
|
kodiVersion = root.attrib.get('version')
|
|
43
|
logging.info('Kodi settings version is: %s', kodiVersion)
|
|
44
|
for setting in root.findall('setting'):
|
|
45
|
if kodiVersion == '2':
|
|
46
|
settingStr = setting.text
|
|
47
|
else:
|
|
48
|
settingStr = setting.get('value')
|
|
49
|
if settingStr == '':
|
|
50
|
settingStr = None
|
|
51
|
settingID = setting.get('id')
|
|
52
|
settingsDict[settingID] = settingStr
|
|
53
|
for setting in settingsDict:
|
|
54
|
if setting == 'slist':
|
|
55
|
stationList = settingsDict[setting]
|
|
56
|
if setting == 'zipcode':
|
|
57
|
zipcode = settingsDict[setting]
|
|
58
|
if setting == 'lineup':
|
|
59
|
lineup = settingsDict[setting]
|
|
60
|
if setting == 'lineupcode':
|
|
61
|
lineupcode = settingsDict[setting]
|
|
62
|
if setting == 'days':
|
|
63
|
days = settingsDict[setting]
|
|
64
|
if setting == 'xdetails':
|
|
65
|
xdetails = settingsDict[setting]
|
|
66
|
if setting == 'xdesc':
|
|
67
|
xdesc = settingsDict[setting]
|
|
68
|
if setting == 'epicon':
|
|
69
|
epicon = settingsDict[setting]
|
|
70
|
if setting == 'epgenre':
|
|
71
|
epgenre = settingsDict[setting]
|
|
72
|
if setting == 'tvhurl':
|
|
73
|
tvhurl = settingsDict[setting]
|
|
74
|
if setting == 'tvhport':
|
|
75
|
tvhport = settingsDict[setting]
|
|
76
|
if setting == 'usern':
|
|
77
|
usern = settingsDict[setting]
|
|
78
|
if setting == 'passw':
|
|
79
|
passw = settingsDict[setting]
|
|
80
|
if setting == 'chmatch':
|
|
81
|
chmatch = settingsDict[setting]
|
|
82
|
if setting == 'tvhmatch':
|
|
83
|
tvhmatch = settingsDict[setting]
|
|
84
|
if setting.startswith('desc'):
|
|
85
|
xdescOrderDict[setting] = (settingsDict[setting])
|
|
86
|
xdescOrder = [value for (key, value) in sorted(xdescOrderDict.items())]
|
|
87
|
if lineupcode != 'lineupId':
|
|
88
|
chmatch = 'false'
|
|
89
|
tvhmatch = 'false'
|
|
90
|
if zipcode.isdigit():
|
|
91
|
country = 'USA'
|
|
92
|
else:
|
|
93
|
country = 'CAN'
|
|
94
|
logging.info('Running zap2epg-0.6.3 for zipcode: %s and lineup: %s', zipcode, lineup)
|
|
95
|
pythonStartTime = time.time()
|
|
96
|
cacheDir = os.path.join(userdata, 'cache')
|
|
97
|
dayHours = int(days) * 8 # set back to 8 when done testing
|
|
98
|
gridtimeStart = (int(time.mktime(time.strptime(str(datetime.datetime.now().replace(microsecond=0,second=0,minute=0)), '%Y-%m-%d %H:%M:%S'))))
|
|
99
|
schedule = {}
|
|
100
|
tvhMatchDict = {}
|
|
101
|
|
|
102
|
def tvhMatchGet():
|
|
103
|
tvhUrlBase = 'http://' + tvhurl + ":" + tvhport
|
|
104
|
channels_url = tvhUrlBase + '/api/channel/grid?all=1&limit=999999999&sort=name&filter=[{"type":"boolean","value":true,"field":"enabled"}]'
|
|
105
|
if usern is not None and passw is not None:
|
|
106
|
logging.info('Adding Tvheadend username and password to request url...')
|
|
107
|
request = urllib2.Request(channels_url)
|
|
108
|
request.add_header('Authorization', b'Basic ' + base64.b64encode(usern + b':' + passw))
|
|
109
|
response = urllib2.urlopen(request)
|
|
110
|
else:
|
|
111
|
response = urllib2.urlopen(channels_url)
|
|
112
|
try:
|
|
113
|
logging.info('Accessing Tvheadend channel list from: %s', tvhUrlBase)
|
|
114
|
channels = json.load(response)
|
|
115
|
for ch in channels['entries']:
|
|
116
|
channelName = ch['name']
|
|
117
|
channelNum = ch['number']
|
|
118
|
tvhMatchDict[channelNum] = channelName
|
|
119
|
logging.info('%s Tvheadend channels found...', str(len(tvhMatchDict)))
|
|
120
|
except urllib2.HTTPError as e:
|
|
121
|
logging.exception('Exception: tvhMatch - %s', e.strerror)
|
|
122
|
pass
|
|
123
|
|
|
124
|
def deleteOldCache(gridtimeStart, showList):
|
|
125
|
logging.info('Checking for old cache files...')
|
|
126
|
try:
|
|
127
|
if os.path.exists(cacheDir):
|
|
128
|
entries = os.listdir(cacheDir)
|
|
129
|
for entry in entries:
|
|
130
|
oldfile = entry.split('.')[0]
|
|
131
|
if oldfile.isdigit():
|
|
132
|
fn = os.path.join(cacheDir, entry)
|
|
133
|
if (int(oldfile) + 10800) < gridtimeStart:
|
|
134
|
try:
|
|
135
|
os.remove(fn)
|
|
136
|
logging.info('Deleting old cache: %s', entry)
|
|
137
|
except OSError, e:
|
|
138
|
logging.warn('Error Deleting: %s - %s.' % (e.filename, e.strerror))
|
|
139
|
elif not oldfile.isdigit():
|
|
140
|
fn = os.path.join(cacheDir, entry)
|
|
141
|
if oldfile not in showList:
|
|
142
|
try:
|
|
143
|
os.remove(fn)
|
|
144
|
logging.info('Deleting old cache: %s', entry)
|
|
145
|
except OSError, e:
|
|
146
|
logging.warn('Error Deleting: %s - %s.' % (e.filename, e.strerror))
|
|
147
|
except Exception as e:
|
|
148
|
logging.exception('Exception: deleteOldCache - %s', e.strerror)
|
|
149
|
|
|
150
|
def convTime(t):
|
|
151
|
return time.strftime("%Y%m%d%H%M%S",time.localtime(int(t)))
|
|
152
|
|
|
153
|
def savepage(fn, data):
|
|
154
|
if not os.path.exists(cacheDir):
|
|
155
|
os.mkdir(cacheDir)
|
|
156
|
fileDir = os.path.join(cacheDir, fn)
|
|
157
|
with gzip.open(fileDir,"wb+") as f:
|
|
158
|
f.write(data)
|
|
159
|
f.close()
|
|
160
|
|
|
161
|
def genreSort(EPfilter, EPgenre):
|
|
162
|
genreList = []
|
|
163
|
if epgenre == '2':
|
|
164
|
# for f in EPfilter:
|
|
165
|
# fClean = re.sub('filter-','',f)
|
|
166
|
# genreList.append(fClean)
|
|
167
|
for g in EPgenre:
|
|
168
|
if g != "Comedy":
|
|
169
|
genreList.append(g)
|
|
170
|
if 'Movie' in genreList or 'movie' in genreList or 'Movies' in genreList:
|
|
171
|
genreList.insert(0, "Movie / Drama")
|
|
172
|
if 'News' in genreList:
|
|
173
|
genreList.insert(0, "News / Current affairs")
|
|
174
|
if 'Game show' in genreList:
|
|
175
|
genreList.insert(0, "Game show / Quiz / Contest")
|
|
176
|
if 'Law' in genreList:
|
|
177
|
genreList.insert(0, "Show / Game show")
|
|
178
|
if 'Art' in genreList or 'Culture' in genreList:
|
|
179
|
genreList.insert(0, "Arts / Culture (without music)")
|
|
180
|
if 'Entertainment' in genreList:
|
|
181
|
genreList.insert(0, "Popular culture / Traditional Arts")
|
|
182
|
if 'Politics' in genreList or 'Social' in genreList or 'Public affairs' in genreList:
|
|
183
|
genreList.insert(0, "Social / Political issues / Economics")
|
|
184
|
if 'Education' in genreList or 'Science' in genreList:
|
|
185
|
genreList.insert(0, "Education / Science / Factual topics")
|
|
186
|
if 'How-to' in genreList:
|
|
187
|
genreList.insert(0, "Leisure hobbies")
|
|
188
|
if 'Travel' in genreList:
|
|
189
|
genreList.insert(0, "Tourism / Travel")
|
|
190
|
if 'Sitcom' in genreList:
|
|
191
|
genreList.insert(0, "Variety show")
|
|
192
|
if 'Talk' in genreList:
|
|
193
|
genreList.insert(0, "Talk show")
|
|
194
|
if 'Children' in genreList:
|
|
195
|
genreList.insert(0, "Children's / Youth programs")
|
|
196
|
if 'Animated' in genreList:
|
|
197
|
genreList.insert(0, "Cartoons / Puppets")
|
|
198
|
if 'Music' in genreList:
|
|
199
|
genreList.insert(0, "Music / Ballet / Dance")
|
|
200
|
if epgenre == '1':
|
|
201
|
# for f in EPfilter:
|
|
202
|
# fClean = re.sub('filter-','',f)
|
|
203
|
# genreList.append(fClean)
|
|
204
|
for g in EPgenre:
|
|
205
|
genreList.append(g)
|
|
206
|
if 'Movie' in genreList or 'movie' in genreList or 'Movies' in genreList:
|
|
207
|
genreList = ["Movie / Drama"]
|
|
208
|
elif 'News' in genreList:
|
|
209
|
genreList = ["News / Current affairs"]
|
|
210
|
elif 'News magazine' in genreList:
|
|
211
|
genreList = ["News magazine"]
|
|
212
|
elif 'Public affairs' in genreList:
|
|
213
|
genreList = ["News / Current affairs"]
|
|
214
|
elif 'Interview' in genreList:
|
|
215
|
genreList = ["Discussion / Interview / Debate"]
|
|
216
|
elif 'Game show' in genreList:
|
|
217
|
genreList = ["Game show / Quiz / Contest"]
|
|
218
|
elif 'Talk' in genreList:
|
|
219
|
genreList = ["Talk show"]
|
|
220
|
elif 'Sports' in genreList:
|
|
221
|
genreList = ["Sports"]
|
|
222
|
elif 'Sitcom' in genreList:
|
|
223
|
genreList = ["Variety show"]
|
|
224
|
elif 'Children' in genreList:
|
|
225
|
genreList = ["Children's / Youth programs"]
|
|
226
|
else:
|
|
227
|
genreList = ["Variety show"]
|
|
228
|
if epgenre == '3':
|
|
229
|
# for f in EPfilter:
|
|
230
|
# fClean = re.sub('filter-','',f)
|
|
231
|
# genreList.append(fClean)
|
|
232
|
for g in EPgenre:
|
|
233
|
genreList.append(g)
|
|
234
|
if 'Movie' in genreList:
|
|
235
|
genreList.remove('Movie')
|
|
236
|
genreList.insert(0, 'Movie')
|
|
237
|
return genreList
|
|
238
|
|
|
239
|
def printHeader(fh, enc):
|
|
240
|
logging.info('Creating xmltv.xml file...')
|
|
241
|
fh.write("<?xml version=\"1.0\" encoding=\""+ enc + "\"?>\n")
|
|
242
|
fh.write("<!DOCTYPE tv SYSTEM \"xmltv.dtd\">\n\n")
|
|
243
|
fh.write("<tv source-info-url=\"http://tvschedule.zap2it.com/\" source-info-name=\"zap2it.com\">\n")
|
|
244
|
|
|
245
|
def printFooter(fh):
|
|
246
|
fh.write("</tv>\n")
|
|
247
|
|
|
248
|
def printStations(fh):
|
|
249
|
global stationCount
|
|
250
|
stationCount = 0
|
|
251
|
try:
|
|
252
|
logging.info('Writing Stations to xmltv.xml file...')
|
|
253
|
try:
|
|
254
|
scheduleSort = OrderedDict(sorted(schedule.iteritems(), key=lambda x: int(x[1]['chnum'])))
|
|
255
|
except:
|
|
256
|
scheduleSort = OrderedDict(sorted(schedule.iteritems(), key=lambda x: x[1]['chfcc']))
|
|
257
|
for station in scheduleSort:
|
|
258
|
fh.write('\t<channel id=\"' + station + '.zap2epg\">\n')
|
|
259
|
if 'chtvh' in scheduleSort[station] and scheduleSort[station]['chtvh'] is not None:
|
|
260
|
xchtvh = re.sub('&','&',scheduleSort[station]['chtvh'])
|
|
261
|
fh.write('\t\t<display-name>' + xchtvh + '</display-name>\n')
|
|
262
|
if 'chnum' in scheduleSort[station] and 'chfcc' in scheduleSort[station]:
|
|
263
|
xchnum = scheduleSort[station]['chnum']
|
|
264
|
xchfcc = scheduleSort[station]['chfcc']
|
|
265
|
fh.write('\t\t<display-name>' + xchnum + ' ' + re.sub('&','&',xchfcc) + '</display-name>\n')
|
|
266
|
fh.write('\t\t<display-name>' + re.sub('&','&',xchfcc) + '</display-name>\n')
|
|
267
|
fh.write('\t\t<display-name>' + xchnum + '</display-name>\n')
|
|
268
|
elif 'chfcc' in scheduleSort[station]:
|
|
269
|
xchnum = scheduleSort[station]['chfcc']
|
|
270
|
fh.write('\t\t<display-name>' + re.sub('&','&',xcfcc) + '</display-name>\n')
|
|
271
|
elif 'chnum' in scheduleSort[station]:
|
|
272
|
xchnum = scheduleSort[station]['chnum']
|
|
273
|
fh.write('\t\t<display-name>' + xchnum + '</display-name>\n')
|
|
274
|
if 'chicon' in scheduleSort[station]:
|
|
275
|
fh.write("\t\t<icon src=\"http:" + scheduleSort[station]['chicon'] + "\" />\n")
|
|
276
|
fh.write("\t</channel>\n")
|
|
277
|
stationCount += 1
|
|
278
|
except Exception as e:
|
|
279
|
logging.exception('Exception: printStations')
|
|
280
|
|
|
281
|
def printEpisodes(fh):
|
|
282
|
global episodeCount
|
|
283
|
episodeCount = 0
|
|
284
|
try:
|
|
285
|
logging.info('Writing Episodes to xmltv.xml file...')
|
|
286
|
if xdesc is True:
|
|
287
|
logging.info('Appending Xdetails to description for xmltv.xml file...')
|
|
288
|
for station in schedule:
|
|
289
|
lang = 'en'
|
|
290
|
sdict = schedule[station]
|
|
291
|
for episode in sdict:
|
|
292
|
if not episode.startswith("ch"):
|
|
293
|
try:
|
|
294
|
edict = sdict[episode]
|
|
295
|
if 'epstart' in edict:
|
|
296
|
startTime = convTime(edict['epstart'])
|
|
297
|
is_dst = time.daylight and time.localtime().tm_isdst > 0
|
|
298
|
TZoffset = "%.2d%.2d" %(- (time.altzone if is_dst else time.timezone)/3600, 0)
|
|
299
|
stopTime = convTime(edict['epend'])
|
|
300
|
fh.write('\t<programme start=\"' + startTime + ' ' + TZoffset + '\" stop=\"' + stopTime + ' ' + TZoffset + '\" channel=\"' + station + '.zap2epg' + '\">\n')
|
|
301
|
fh.write('\t\t<episode-num system=\"dd_progid\">' + edict['epid'] + '</episode-num>\n')
|
|
302
|
if edict['epshow'] is not None:
|
|
303
|
fh.write('\t\t<title lang=\"' + lang + '\">' + re.sub('&','&',edict['epshow']) + '</title>\n')
|
|
304
|
if edict['eptitle'] is not None:
|
|
305
|
fh.write('\t\t<sub-title lang=\"'+ lang + '\">' + re.sub('&','&', edict['eptitle']) + '</sub-title>\n')
|
|
306
|
if xdesc == 'true':
|
|
307
|
xdescSort = addXDetails(edict)
|
|
308
|
fh.write('\t\t<desc lang=\"' + lang + '\">' + re.sub('&','&', xdescSort) + '</desc>\n')
|
|
309
|
if xdesc == 'false':
|
|
310
|
if edict['epdesc'] is not None:
|
|
311
|
fh.write('\t\t<desc lang=\"' + lang + '\">' + re.sub('&','&', edict['epdesc']) + '</desc>\n')
|
|
312
|
if edict['epsn'] is not None and edict['epen'] is not None:
|
|
313
|
fh.write("\t\t<episode-num system=\"onscreen\">" + 'S' + edict['epsn'].zfill(2) + 'E' + edict['epen'].zfill(2) + "</episode-num>\n")
|
|
314
|
fh.write("\t\t<episode-num system=\"xmltv_ns\">" + str(int(edict['epsn'])-1) + "." + str(int(edict['epen'])-1) + ".</episode-num>\n")
|
|
315
|
if edict['epyear'] is not None:
|
|
316
|
fh.write('\t\t<date>' + edict['epyear'] + '</date>\n')
|
|
317
|
if not episode.startswith("MV"):
|
|
318
|
if epicon == '1':
|
|
319
|
if edict['epimage'] is not None and edict['epimage'] != '':
|
|
320
|
fh.write('\t\t<icon src="https://zap2it.tmsimg.com/assets/' + edict['epimage'] + '.jpg" />\n')
|
|
321
|
else:
|
|
322
|
if edict['epthumb'] is not None and edict['epthumb'] != '':
|
|
323
|
fh.write('\t\t<icon src="https://zap2it.tmsimg.com/assets/' + edict['epthumb'] + '.jpg" />\n')
|
|
324
|
if epicon == '2':
|
|
325
|
if edict['epthumb'] is not None and edict['epthumb'] != '':
|
|
326
|
fh.write('\t\t<icon src="https://zap2it.tmsimg.com/assets/' + edict['epthumb'] + '.jpg" />\n')
|
|
327
|
if episode.startswith("MV"):
|
|
328
|
if edict['epthumb'] is not None and edict['epthumb'] != '':
|
|
329
|
fh.write('\t\t<icon src="https://zap2it.tmsimg.com/assets/' + edict['epthumb'] + '.jpg" />\n')
|
|
330
|
if not any(i in ['New', 'Live'] for i in edict['epflag']):
|
|
331
|
fh.write("\t\t<previously-shown ")
|
|
332
|
if edict['epoad'] is not None and int(edict['epoad']) > 0:
|
|
333
|
fh.write("start=\"" + convTime(edict['epoad']) + " " + TZoffset + "\"")
|
|
334
|
fh.write(" />\n")
|
|
335
|
if edict['epflag'] is not None:
|
|
336
|
if 'New' in edict['epflag']:
|
|
337
|
fh.write("\t\t<new />\n")
|
|
338
|
if 'Live' in edict['epflag']:
|
|
339
|
fh.write("\t\t<live />\n")
|
|
340
|
if edict['eprating'] is not None:
|
|
341
|
fh.write('\t\t<rating>\n\t\t\t<value>' + edict['eprating'] + '</value>\n\t\t</rating>\n')
|
|
342
|
if edict['epstar'] is not None:
|
|
343
|
fh.write('\t\t<star-rating>\n\t\t\t<value>' + edict['epstar'] + '/4</value>\n\t\t</star-rating>\n')
|
|
344
|
if epgenre != '0':
|
|
345
|
if edict['epfilter'] is not None and edict['epgenres'] is not None:
|
|
346
|
genreNewList = genreSort(edict['epfilter'], edict['epgenres'])
|
|
347
|
for genre in genreNewList:
|
|
348
|
fh.write("\t\t<category lang=\"" + lang + "\">" + genre + "</category>\n")
|
|
349
|
fh.write("\t</programme>\n")
|
|
350
|
episodeCount += 1
|
|
351
|
except Exception as e:
|
|
352
|
logging.exception('No data for episode %s:', episode)
|
|
353
|
#fn = os.path.join(cacheDir, episode + '.json')
|
|
354
|
#os.remove(fn)
|
|
355
|
#logging.info('Deleting episode %s:', episode)
|
|
356
|
except Exception as e:
|
|
357
|
logging.exception('Exception: printEpisodes')
|
|
358
|
|
|
359
|
def xmltv():
|
|
360
|
try:
|
|
361
|
enc = 'utf-8'
|
|
362
|
outFile = os.path.join(userdata, 'xmltv.xml')
|
|
363
|
fh = codecs.open(outFile, 'w+b', encoding=enc)
|
|
364
|
printHeader(fh, enc)
|
|
365
|
printStations(fh)
|
|
366
|
printEpisodes(fh)
|
|
367
|
printFooter(fh)
|
|
368
|
fh.close()
|
|
369
|
except Exception as e:
|
|
370
|
logging.exception('Exception: xmltv')
|
|
371
|
|
|
372
|
def parseStations(content):
|
|
373
|
try:
|
|
374
|
ch_guide = json.loads(content)
|
|
375
|
for station in ch_guide['channels']:
|
|
376
|
skey = station.get('channelId')
|
|
377
|
if stationList is not None:
|
|
378
|
if skey in stationList:
|
|
379
|
schedule[skey] = {}
|
|
380
|
chName = station.get('callSign')
|
|
381
|
schedule[skey]['chfcc'] = chName
|
|
382
|
schedule[skey]['chicon'] = station.get('thumbnail').split('?')[0]
|
|
383
|
chnumStart = station.get('channelNo')
|
|
384
|
if '.' not in chnumStart and chmatch == 'true' and chName is not None:
|
|
385
|
chsub = re.search('(\d+)$', chName)
|
|
386
|
if chsub is not None:
|
|
387
|
chnumUpdate = chnumStart + '.' + chsub.group(0)
|
|
388
|
else:
|
|
389
|
chnumUpdate = chnumStart + '.1'
|
|
390
|
else:
|
|
391
|
chnumUpdate = chnumStart
|
|
392
|
schedule[skey]['chnum'] = chnumUpdate
|
|
393
|
if tvhmatch == 'true' and '.' in chnumUpdate:
|
|
394
|
if chnumUpdate in tvhMatchDict:
|
|
395
|
schedule[skey]['chtvh'] = tvhMatchDict[chnumUpdate]
|
|
396
|
else:
|
|
397
|
schedule[skey]['chtvh'] = None
|
|
398
|
else:
|
|
399
|
schedule[skey] = {}
|
|
400
|
chName = station.get('callSign')
|
|
401
|
schedule[skey]['chfcc'] = chName
|
|
402
|
schedule[skey]['chicon'] = station.get('thumbnail').split('?')[0]
|
|
403
|
chnumStart = station.get('channelNo')
|
|
404
|
if '.' not in chnumStart and chmatch == 'true' and chName is not None:
|
|
405
|
chsub = re.search('(\d+)$', chName)
|
|
406
|
if chsub is not None:
|
|
407
|
chnumUpdate = chnumStart + '.' + chsub.group(0)
|
|
408
|
else:
|
|
409
|
chnumUpdate = chnumStart + '.1'
|
|
410
|
else:
|
|
411
|
chnumUpdate = chnumStart
|
|
412
|
schedule[skey]['chnum'] = chnumUpdate
|
|
413
|
if tvhmatch == 'true' and '.' in chnumUpdate:
|
|
414
|
if chnumUpdate in tvhMatchDict:
|
|
415
|
schedule[skey]['chtvh'] = tvhMatchDict[chnumUpdate]
|
|
416
|
else:
|
|
417
|
schedule[skey]['chtvh'] = None
|
|
418
|
except Exception as e:
|
|
419
|
logging.exception('Exception: parseStations')
|
|
420
|
|
|
421
|
def parseEpisodes(content):
|
|
422
|
try:
|
|
423
|
ch_guide = json.loads(content)
|
|
424
|
for station in ch_guide['channels']:
|
|
425
|
skey = station.get('channelId')
|
|
426
|
if stationList is not None:
|
|
427
|
if skey in stationList:
|
|
428
|
episodes = station.get('events')
|
|
429
|
for episode in episodes:
|
|
430
|
epkey = str(calendar.timegm(time.strptime(episode.get('startTime'), '%Y-%m-%dT%H:%M:%SZ')))
|
|
431
|
schedule[skey][epkey] = {}
|
|
432
|
schedule[skey][epkey]['epid'] = episode['program'].get('tmsId')
|
|
433
|
schedule[skey][epkey]['epstart'] = str(calendar.timegm(time.strptime(episode.get('startTime'), '%Y-%m-%dT%H:%M:%SZ')))
|
|
434
|
schedule[skey][epkey]['epend'] = str(calendar.timegm(time.strptime(episode.get('endTime'), '%Y-%m-%dT%H:%M:%SZ')))
|
|
435
|
schedule[skey][epkey]['eplength'] = episode.get('duration')
|
|
436
|
schedule[skey][epkey]['epshow'] = episode['program'].get('title')
|
|
437
|
schedule[skey][epkey]['eptitle'] = episode['program'].get('episodeTitle')
|
|
438
|
schedule[skey][epkey]['epdesc'] = episode['program'].get('shortDesc')
|
|
439
|
schedule[skey][epkey]['epyear'] = episode['program'].get('releaseYear')
|
|
440
|
schedule[skey][epkey]['eprating'] = episode.get('rating')
|
|
441
|
schedule[skey][epkey]['epflag'] = episode.get('flag')
|
|
442
|
schedule[skey][epkey]['eptags'] = episode.get('tags')
|
|
443
|
schedule[skey][epkey]['epsn'] = episode['program'].get('season')
|
|
444
|
schedule[skey][epkey]['epen'] = episode['program'].get('episode')
|
|
445
|
schedule[skey][epkey]['epthumb'] = episode.get('thumbnail')
|
|
446
|
schedule[skey][epkey]['epoad'] = None
|
|
447
|
schedule[skey][epkey]['epstar'] = None
|
|
448
|
schedule[skey][epkey]['epfilter'] = episode.get('filter')
|
|
449
|
schedule[skey][epkey]['epgenres'] = None
|
|
450
|
schedule[skey][epkey]['epcredits'] = None
|
|
451
|
schedule[skey][epkey]['epxdesc'] = None
|
|
452
|
schedule[skey][epkey]['epseries'] = episode.get('seriesId')
|
|
453
|
schedule[skey][epkey]['epimage'] = None
|
|
454
|
schedule[skey][epkey]['epfan'] = None
|
|
455
|
else:
|
|
456
|
episodes = station.get('events')
|
|
457
|
for episode in episodes:
|
|
458
|
epkey = str(calendar.timegm(time.strptime(episode.get('startTime'), '%Y-%m-%dT%H:%M:%SZ')))
|
|
459
|
schedule[skey][epkey] = {}
|
|
460
|
schedule[skey][epkey]['epid'] = episode['program'].get('tmsId')
|
|
461
|
schedule[skey][epkey]['epstart'] = str(calendar.timegm(time.strptime(episode.get('startTime'), '%Y-%m-%dT%H:%M:%SZ')))
|
|
462
|
schedule[skey][epkey]['epend'] = str(calendar.timegm(time.strptime(episode.get('endTime'), '%Y-%m-%dT%H:%M:%SZ')))
|
|
463
|
schedule[skey][epkey]['eplength'] = episode.get('duration')
|
|
464
|
schedule[skey][epkey]['epshow'] = episode['program'].get('title')
|
|
465
|
schedule[skey][epkey]['eptitle'] = episode['program'].get('episodeTitle')
|
|
466
|
schedule[skey][epkey]['epdesc'] = episode['program'].get('shortDesc')
|
|
467
|
schedule[skey][epkey]['epyear'] = episode['program'].get('releaseYear')
|
|
468
|
schedule[skey][epkey]['eprating'] = episode.get('rating')
|
|
469
|
schedule[skey][epkey]['epflag'] = episode.get('flag')
|
|
470
|
schedule[skey][epkey]['eptags'] = episode.get('tags')
|
|
471
|
schedule[skey][epkey]['epsn'] = episode['program'].get('season')
|
|
472
|
schedule[skey][epkey]['epen'] = episode['program'].get('episode')
|
|
473
|
schedule[skey][epkey]['epthumb'] = episode.get('thumbnail')
|
|
474
|
schedule[skey][epkey]['epoad'] = None
|
|
475
|
schedule[skey][epkey]['epstar'] = None
|
|
476
|
schedule[skey][epkey]['epfilter'] = episode.get('filter')
|
|
477
|
schedule[skey][epkey]['epgenres'] = None
|
|
478
|
schedule[skey][epkey]['epcredits'] = None
|
|
479
|
schedule[skey][epkey]['epxdesc'] = None
|
|
480
|
schedule[skey][epkey]['epseries'] = episode.get('seriesId')
|
|
481
|
schedule[skey][epkey]['epimage'] = None
|
|
482
|
schedule[skey][epkey]['epfan'] = None
|
|
483
|
except Exception as e:
|
|
484
|
logging.exception('Exception: parseEpisodes')
|
|
485
|
|
|
486
|
def parseXdetails():
|
|
487
|
showList = []
|
|
488
|
failList = []
|
|
489
|
try:
|
|
490
|
for station in schedule:
|
|
491
|
sdict = schedule[station]
|
|
492
|
for episode in sdict:
|
|
493
|
if not episode.startswith("ch"):
|
|
494
|
edict = sdict[episode]
|
|
495
|
EPseries = edict['epseries']
|
|
496
|
showList.append(edict['epseries'])
|
|
497
|
filename = EPseries + '.json'
|
|
498
|
fileDir = os.path.join(cacheDir, filename)
|
|
499
|
try:
|
|
500
|
if not os.path.exists(fileDir) and EPseries not in failList:
|
|
501
|
retry = 3
|
|
502
|
while retry > 0:
|
|
503
|
logging.info('Downloading details data for: %s', EPseries)
|
|
504
|
url = 'https://tvlistings.gracenote.com/api/program/overviewDetails'
|
|
505
|
data = 'programSeriesID=' + EPseries
|
|
506
|
try:
|
|
507
|
URLcontent = urllib2.Request(url, data=data)
|
|
508
|
JSONcontent = urllib2.urlopen(URLcontent).read()
|
|
509
|
if JSONcontent:
|
|
510
|
with open(fileDir,"wb+") as f:
|
|
511
|
f.write(JSONcontent)
|
|
512
|
f.close()
|
|
513
|
retry = 0
|
|
514
|
else:
|
|
515
|
time.sleep(1)
|
|
516
|
retry -= 1
|
|
517
|
logging.warn('Retry downloading missing details data for: %s', EPseries)
|
|
518
|
except urllib2.URLError, e:
|
|
519
|
time.sleep(1)
|
|
520
|
retry -= 1
|
|
521
|
logging.warn('Retry downloading details data for: %s - %s', EPseries, e)
|
|
522
|
if os.path.exists(fileDir):
|
|
523
|
fileSize = os.path.getsize(fileDir)
|
|
524
|
if fileSize > 0:
|
|
525
|
with open(fileDir, 'rb') as f:
|
|
526
|
EPdetails = json.loads(f.read())
|
|
527
|
f.close()
|
|
528
|
logging.info('Parsing %s', filename)
|
|
529
|
edict['epimage'] = EPdetails.get('seriesImage')
|
|
530
|
edict['epfan'] = EPdetails.get('backgroundImage')
|
|
531
|
EPgenres = EPdetails.get('seriesGenres')
|
|
532
|
if filename.startswith("MV"):
|
|
533
|
edict['epcredits'] = EPdetails['overviewTab'].get('cast')
|
|
534
|
EPgenres = 'Movie|' + EPgenres
|
|
535
|
edict['epgenres'] = EPgenres.split('|')
|
|
536
|
#edict['epstar'] = EPdetails.get('starRating')
|
|
537
|
EPlist = EPdetails['upcomingEpisodeTab']
|
|
538
|
EPid = edict['epid']
|
|
539
|
for airing in EPlist:
|
|
540
|
if EPid.lower() == airing['tmsID'].lower():
|
|
541
|
if not episode.startswith("MV"):
|
|
542
|
try:
|
|
543
|
origDate = airing.get('originalAirDate')
|
|
544
|
if origDate != '':
|
|
545
|
EPoad = re.sub('Z', ':00Z', airing.get('originalAirDate'))
|
|
546
|
edict['epoad'] = str(calendar.timegm(time.strptime(EPoad, '%Y-%m-%dT%H:%M:%SZ')))
|
|
547
|
except Exception as e:
|
|
548
|
logging.exception('Could not parse oad for: %s - %s', episode, e)
|
|
549
|
|
|
550
|
else:
|
|
551
|
logging.warn('Could not parse data for: %s - deleting file', filename)
|
|
552
|
os.remove(fileDir)
|
|
553
|
else:
|
|
554
|
logging.warn('Could not download details data for: %s - skipping episode', episode)
|
|
555
|
failList.append(EPseries)
|
|
556
|
except Exception as e:
|
|
557
|
logging.exception('Could not parse data for: %s - deleting file - %s', episode, e)
|
|
558
|
#os.remove(fileDir)
|
|
559
|
except Exception as e:
|
|
560
|
logging.exception('Exception: parseXdetails')
|
|
561
|
return showList
|
|
562
|
|
|
563
|
def addXDetails(edict):
|
|
564
|
try:
|
|
565
|
ratings = ""
|
|
566
|
date = ""
|
|
567
|
myear = ""
|
|
568
|
new = ""
|
|
569
|
live = ""
|
|
570
|
hd = ""
|
|
571
|
cc = ""
|
|
572
|
cast = ""
|
|
573
|
season = ""
|
|
574
|
epis = ""
|
|
575
|
episqts = ""
|
|
576
|
prog = ""
|
|
577
|
plot= ""
|
|
578
|
descsort = ""
|
|
579
|
bullet = u"\u2022 "
|
|
580
|
hyphen = u"\u2013 "
|
|
581
|
newLine = "\n"
|
|
582
|
space = " "
|
|
583
|
colon = u"\u003A "
|
|
584
|
vbar = u"\u007C "
|
|
585
|
slash = u"\u2215 "
|
|
586
|
comma = u"\u002C "
|
|
587
|
|
|
588
|
def getSortName(opt):
|
|
589
|
return {
|
|
590
|
1: bullet,
|
|
591
|
2: newLine,
|
|
592
|
3: hyphen,
|
|
593
|
4: space,
|
|
594
|
5: colon,
|
|
595
|
6: vbar,
|
|
596
|
7: slash,
|
|
597
|
8: comma,
|
|
598
|
9: plot,
|
|
599
|
10: new,
|
|
600
|
11: hd,
|
|
601
|
12: cc,
|
|
602
|
13: season,
|
|
603
|
14: ratings,
|
|
604
|
15: date,
|
|
605
|
16: prog,
|
|
606
|
17: epis,
|
|
607
|
18: episqts,
|
|
608
|
19: cast,
|
|
609
|
20: myear,
|
|
610
|
}.get(opt, None)
|
|
611
|
|
|
612
|
def cleanSortList(optList):
|
|
613
|
cleanList=[]
|
|
614
|
optLen = len(optList)
|
|
615
|
for opt in optList:
|
|
616
|
thisOption = getSortName(int(opt))
|
|
617
|
if thisOption:
|
|
618
|
cleanList.append(int(opt))
|
|
619
|
for item in reversed(cleanList):
|
|
620
|
if cleanList[-1] <= 8:
|
|
621
|
del cleanList[-1]
|
|
622
|
return cleanList
|
|
623
|
|
|
624
|
def makeDescsortList(optList):
|
|
625
|
sortOrderList =[]
|
|
626
|
lastOption = 1
|
|
627
|
cleanedList = cleanSortList(optList)
|
|
628
|
for opt in cleanedList:
|
|
629
|
thisOption = getSortName(int(opt))
|
|
630
|
if int(opt) <= 8 and lastOption <= 8:
|
|
631
|
if int(opt) == 2 and len(sortOrderList) > 1:
|
|
632
|
del sortOrderList[-1]
|
|
633
|
sortOrderList.append(thisOption)
|
|
634
|
lastOption = int(opt)
|
|
635
|
elif thisOption and lastOption:
|
|
636
|
sortOrderList.append(thisOption)
|
|
637
|
lastOption = int(opt)
|
|
638
|
elif thisOption:
|
|
639
|
lastOption = int(opt)
|
|
640
|
return sortOrderList
|
|
641
|
|
|
642
|
if edict['epoad'] is not None and int(edict['epoad']) > 0:
|
|
643
|
is_dst = time.daylight and time.localtime().tm_isdst > 0
|
|
644
|
TZoffset = (time.altzone if is_dst else time.timezone)
|
|
645
|
origDate = int(edict['epoad']) + TZoffset
|
|
646
|
finalDate = datetime.datetime.fromtimestamp(origDate).strftime('%B %d%% %Y')
|
|
647
|
finalDate = re.sub('%', ',', finalDate)
|
|
648
|
date = "First aired: " + finalDate + space
|
|
649
|
if edict['epyear'] is not None:
|
|
650
|
myear = "Released: " + edict['epyear'] + space
|
|
651
|
if edict['eprating'] is not None:
|
|
652
|
ratings = edict['eprating'] + space
|
|
653
|
if edict['epflag'] != []:
|
|
654
|
flagList = edict['epflag']
|
|
655
|
new = ' - '.join(flagList).upper() + space
|
|
656
|
#if edict['epnew'] is not None:
|
|
657
|
#new = edict['epnew'] + space
|
|
658
|
#if edict['eplive'] is not None:
|
|
659
|
#new = edict['eplive'] + space
|
|
660
|
#if edict['epprem'] is not None:
|
|
661
|
#new = edict['epprem'] + space
|
|
662
|
#if edict['epfin'] is not None:
|
|
663
|
#new = edict['epfin'] + space
|
|
664
|
if edict['eptags'] != []:
|
|
665
|
tagsList = edict['eptags']
|
|
666
|
cc = ' | '.join(tagsList).upper() + space
|
|
667
|
#if edict['ephd'] is not None:
|
|
668
|
#hd = edict['ephd'] + space
|
|
669
|
if edict['epsn'] is not None and edict['epen'] is not None:
|
|
670
|
s = re.sub('S', '', edict['epsn'])
|
|
671
|
sf = "Season " + str(int(s))
|
|
672
|
e = re.sub('E', '', edict['epen'])
|
|
673
|
ef = "Episode " + str(int(e))
|
|
674
|
season = sf + " - " + ef + space
|
|
675
|
if edict['epcredits'] is not None:
|
|
676
|
cast = "Cast: "
|
|
677
|
castlist = ""
|
|
678
|
prev = None
|
|
679
|
EPcastList = []
|
|
680
|
for c in edict['epcredits']:
|
|
681
|
EPcastList.append(c['name'])
|
|
682
|
for g in EPcastList:
|
|
683
|
if prev is None:
|
|
684
|
castlist = g
|
|
685
|
prev = g
|
|
686
|
else:
|
|
687
|
castlist = castlist + ", " + g
|
|
688
|
cast = cast + castlist + space
|
|
689
|
if edict['epshow'] is not None:
|
|
690
|
prog = edict['epshow'] + space
|
|
691
|
if edict['eptitle'] is not None:
|
|
692
|
epis = edict['eptitle'] + space
|
|
693
|
episqts = '\"' + edict['eptitle'] + '\"' + space
|
|
694
|
if edict['epdesc'] is not None:
|
|
695
|
plot = edict['epdesc'] + space
|
|
696
|
|
|
697
|
# todo - handle star ratings
|
|
698
|
|
|
699
|
descsort = "".join(makeDescsortList(xdescOrder))
|
|
700
|
return descsort
|
|
701
|
except Exception as e:
|
|
702
|
logging.exception('Exception: addXdetails to description')
|
|
703
|
|
|
704
|
|
|
705
|
try:
|
|
706
|
if not os.path.exists(cacheDir):
|
|
707
|
os.mkdir(cacheDir)
|
|
708
|
count = 0
|
|
709
|
gridtime = gridtimeStart
|
|
710
|
if stationList is None:
|
|
711
|
logging.info('No channel list found - adding all stations!')
|
|
712
|
if tvhmatch == 'true':
|
|
713
|
tvhMatchGet()
|
|
714
|
while count < dayHours:
|
|
715
|
filename = str(gridtime) + '.json.gz'
|
|
716
|
fileDir = os.path.join(cacheDir, filename)
|
|
717
|
if not os.path.exists(fileDir):
|
|
718
|
try:
|
|
719
|
logging.info('Downloading guide data for: %s', str(gridtime))
|
|
720
|
url = 'http://tvlistings.gracenote.com/api/grid?lineupId=×pan=3&headendId=' + lineupcode + '&country=' + country + '&device=-&postalCode=' + zipcode + '&time=' + str(gridtime) + '&pref=-&userId=-'
|
|
721
|
saveContent = urllib2.urlopen(url).read()
|
|
722
|
savepage(fileDir, saveContent)
|
|
723
|
except:
|
|
724
|
logging.warn('Could not download guide data for: %s', str(gridtime))
|
|
725
|
logging.warn('URL: %s', url)
|
|
726
|
if os.path.exists(fileDir):
|
|
727
|
try:
|
|
728
|
with gzip.open(fileDir, 'rb') as f:
|
|
729
|
content = f.read()
|
|
730
|
f.close()
|
|
731
|
logging.info('Parsing %s', filename)
|
|
732
|
if count == 0:
|
|
733
|
parseStations(content)
|
|
734
|
parseEpisodes(content)
|
|
735
|
except:
|
|
736
|
logging.warn('JSON file error for: %s - deleting file', filename)
|
|
737
|
os.remove(fileDir)
|
|
738
|
count += 1
|
|
739
|
gridtime = gridtime + 10800
|
|
740
|
if xdetails == 'true':
|
|
741
|
showList = parseXdetails()
|
|
742
|
else:
|
|
743
|
showList = []
|
|
744
|
xmltv()
|
|
745
|
deleteOldCache(gridtimeStart, showList)
|
|
746
|
timeRun = round((time.time() - pythonStartTime),2)
|
|
747
|
logging.info('zap2epg completed in %s seconds. ', timeRun)
|
|
748
|
logging.info('%s Stations and %s Episodes written to xmltv.xml file.', str(stationCount), str(episodeCount))
|
|
749
|
####### remove this block after testing
|
|
750
|
# dictFileName = 'REdict.json'
|
|
751
|
# DictFileDir = os.path.join(userdata, dictFileName)
|
|
752
|
# with open(DictFileDir, 'w') as fp:
|
|
753
|
# json.dump(REdict, fp)
|
|
754
|
# dictFileNameTxt = 'schedule.txt'
|
|
755
|
# DictFileTxtDir = os.path.join(userdata, dictFileNameTxt)
|
|
756
|
# f = open(DictFileTxtDir,"w")
|
|
757
|
# f.write( str(schedule) )
|
|
758
|
# f.close()
|
|
759
|
####### remove this block after testing
|
|
760
|
return timeRun, stationCount, episodeCount
|
|
761
|
except Exception as e:
|
|
762
|
logging.exception('Exception: main')
|
|
763
|
|
|
764
|
if __name__ == '__main__':
|
|
765
|
userdata = os.getcwd()
|
|
766
|
log = os.path.join(userdata, 'zap2epg.log')
|
|
767
|
logging.basicConfig(filename=log, filemode='w', format='%(asctime)s %(message)s', datefmt='%Y/%m/%d %H:%M:%S', level=logging.DEBUG)
|
|
768
|
mainRun(userdata)
|