import org.apache.commons.lang.StringEscapeUtils
import org.serviio.library.metadata.*
import org.serviio.library.online.*
import org.serviio.util.*
/**
* Content URL extractor plugin for ARD-Mediathek
*
* @author OAR
*
* Version : 2.0
*/
class ARD extends WebResourceUrlExtractor {
final EXTRACTOR_NAME = 'ARD-Mediathek'
final EXTRACTOR_VERSION = 2
final EXTRACTOR_TIMEOUT = 30
final USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36'
final URL_BASE = 'http://www.ardmediathek.de'
final URL_ASSETBASE = URL_BASE + '/play/media/%s?devicetype=pc'
final DATE_FORMAT = 'dd.MM.yyyy HH:mm'
final KEY_VIDEO = 'VIDEO'
final KEY_THUMB = 'THUMB'
final KEY_EXPIRE = 'EXPIRE'
final KEY_ASSET_HIGH = 'ASSET_H'
final KEY_ASSET_MED = 'ASSET_M'
final KEY_ASSET_LOW = 'ASSET_L'
final VALID_FEED_URL = '^http[s]?://www\\.ardmediathek\\.de/.+'
final THEME_TITLE = '(?ms).*?([^<]*)'
final THEME_PAGING = '(?ms)[^<]*\\s* wriCache = new HashMap()
@Override
String getExtractorName() {
return EXTRACTOR_NAME
}
@Override
int getVersion() {
return EXTRACTOR_VERSION
}
@Override
boolean extractorMatches(URL feedUrl) {
return feedUrl ==~ VALID_FEED_URL
}
@Override
int getExtractItemsTimeout() {
return EXTRACTOR_TIMEOUT
}
@Override
WebResourceContainer extractItems(URL resourceUrl, int maxItemsToRetrieve) {
// check wriCache for outdated entries
def now = System.currentTimeMillis()
def nowDate = new Date(now)
def List toDelete = []
wriCache.each {
def itemExpireDate = Date.parse(DATE_FORMAT, it.value.getAdditionalInfo().get(KEY_EXPIRE))
if (itemExpireDate < nowDate) toDelete.add(it.key)
}
toDelete.each {
wriCache.remove(it)
}
def expireDate = new Date(now + getOnlineFeedExpiryInterval()*3600000)
//def expireDate = new Date(now + 48*3600000) // debug
def expire = expireDate.format(DATE_FORMAT)
def webPage
def matcher
def title
def lastReq = ''
def List list = []
def requestURL = resourceUrl
def tryRequest = true
while (tryRequest) {
try {
webPage = openURL(requestURL, USER_AGENT)
}
catch (ex) {
log(ex.toString())
return null
}
// theme title
if (title == null) {
matcher = webPage =~ THEME_TITLE
title = StringEscapeUtils.unescapeHtml(matcher[0][1].toString())
}
// items
matcher = webPage =~ THEME_ITEMS
for (def i = 0; i < matcher.getCount() && list.size() < maxItemsToRetrieve; i++) {
def itemLink = StringEscapeUtils.unescapeHtml(matcher[i][1].toString())
def matcher1 = itemLink =~ THEME_ITEM_ID
if (!matcher1.find()) continue
def id = matcher1[0][1].toString()
if (wriCache.containsKey(id)) {
list.add(wriCache.get(id))
} else {
def item = buildItem(id, matcher[i], expire)
if (item != null) {
wriCache.put(id, item)
list.add(item)
}
}
}
//check for next page
tryRequest = false
if (list.size() < maxItemsToRetrieve) {
matcher = webPage =~ THEME_PAGING
if(matcher.find()) {
def nextReq = StringEscapeUtils.unescapeHtml(matcher[0][1].toString())
requestURL = new URL(URL_BASE + nextReq)
tryRequest = true
}
}
}
// container
def container = new WebResourceContainer()
container.setTitle(title)
container.setThumbnailUrl(THEME_ICON)
container.setItems(list)
return container
}
@Override
ContentURLContainer extractUrl(WebResourceItem item, PreferredQuality requestedQuality) {
def info = item.getAdditionalInfo()
def videoURL
def match
def mapKey
switch (requestedQuality) {
case PreferredQuality.HIGH:
match = VIDEO_HIGH
mapKey = KEY_ASSET_HIGH
break
case PreferredQuality.MEDIUM:
match = VIDEO_MED
mapKey = KEY_ASSET_MED
break
default:
match = VIDEO_LOW
mapKey = KEY_ASSET_LOW
break
}
if (info.containsKey(mapKey))
videoURL = info.get(mapKey)
else {
def link = info.get(KEY_VIDEO)
def asset
try {
asset = openURL(new URL(link), USER_AGENT)
}
catch (ex) {
log(ex.toString())
return null
}
videoURL = getVideoUrl(asset, match)
if (videoURL == null) return null
info.put(mapKey, videoURL)
}
def container = new ContentURLContainer()
container.setContentUrl(videoURL)
container.setThumbnailUrl(info.get(KEY_THUMB))
container.setFileType(MediaFileType.VIDEO)
def expire = info.get(KEY_EXPIRE)
container.setExpiresOn(Date.parse(DATE_FORMAT, expire))
container.setCacheKey(item.getCacheKey() + mapKey)
container.setLive(false)
container.setUserAgent(USER_AGENT)
return container
}
private WebResourceItem buildItem(String id, List matches, String expire) {
// metadata
def title = StringEscapeUtils.unescapeHtml(matches[3].toString())
def assetURL = String.format(URL_ASSETBASE, id)
def itemThumb = StringEscapeUtils.unescapeHtml(matches[2].toString())
def matcher = itemThumb =~ ITEM_THUMB
def thumb = URL_BASE + matcher[0][1].toString() + ITEM_THUMB_SIZE
def map = new HashMap()
map.put(KEY_VIDEO, assetURL)
map.put(KEY_THUMB, thumb)
map.put(KEY_EXPIRE, expire)
def item = new WebResourceItem()
item.setTitle(title)
item.setAdditionalInfo(map)
def parts = StringEscapeUtils.unescapeHtml(matches[4].toString()).split(/\s*\|\s*/)
def release = parts[0]
try {
if (parts[1].endsWith(' Uhr'))
release += ' ' + parts[1].substring(0, parts[1].length() - 4)
else
release += ' 00:00'
def releaseDate = Date.parse(DATE_FORMAT, release)
item.setReleaseDate(releaseDate)
}
catch (ex) {}
item.setCacheKey(ITEM_CACHEPREFIX + id)
return item
}
private String getVideoUrl(String asset, String check) {
def videoURL
def matcher = asset =~ check
if (matcher.find()) videoURL = matcher[0][1].toString()
return videoURL
}
static void main(args) {
// this is just to test
ARD extractor = new ARD();
URL videoLink = new URL("http://www.ardmediathek.de/tv/suche?searchText=arte");
println "Name : " + extractor.getExtractorName();
println "Version : " + extractor.getVersion();
println "TestMatch : " + extractor.extractorMatches(videoLink);
WebResourceContainer container = extractor.extractItems(videoLink, 50);
for (def i=0; i < container.getItems().size(); i++) {
extractor.extractUrl(container.getItems()[i], PreferredQuality.HIGH); println "**** HIGH ****"
}
extractor.extractUrl(container.getItems()[0], PreferredQuality.MEDIUM);println "**** MEDIUM ****"
extractor.extractUrl(container.getItems()[0], PreferredQuality.LOW);println "**** LOW ****"
// Test Cache
container = extractor.extractItems(videoLink, 10);
extractor.extractUrl(container.getItems()[0], PreferredQuality.HIGH);println "**** HIGH ****"
}
}