develop #12

Merged
itdominator merged 6 commits from develop into master 2023-09-30 19:36:35 +00:00
10 changed files with 689 additions and 2 deletions
Showing only changes of commit 153cc450d9 - Show all commits

View File

@ -12,7 +12,7 @@ Copy the share/solarfm folder to your user .config/ directory too.
<h6>Install Setup</h6> <h6>Install Setup</h6>
``` ```
sudo apt-get install python3.8 python3-setproctitle python3-gi wget ffmpegthumbnailer steamcmd sudo apt-get install xclip python3.8 python3-setproctitle python3-gi wget ffmpegthumbnailer steamcmd
``` ```
# Known Issues # Known Issues

View File

@ -0,0 +1,3 @@
"""
Pligin Module
"""

View File

@ -0,0 +1,17 @@
def get_imdb_id(uniqueids):
imdb_id = uniqueids.get('imdb')
if not imdb_id or not imdb_id.startswith('tt'):
return None
return imdb_id
# example format for scraper results
_ScraperResults = {
'info',
'ratings',
'uniqueids',
'cast',
'available_art',
'error',
'warning' # not handled
}

View File

@ -0,0 +1,75 @@
# coding: utf-8
#
# Copyright (C) 2020, Team Kodi
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""Functions to interact with various web site APIs."""
from __future__ import absolute_import, unicode_literals
import json
# from pprint import pformat
try: #PY2 / PY3
from urllib2 import Request, urlopen
from urllib2 import URLError
from urllib import urlencode
except ImportError:
from urllib.request import Request, urlopen
from urllib.error import URLError
from urllib.parse import urlencode
try:
from typing import Text, Optional, Union, List, Dict, Any # pylint: disable=unused-import
InfoType = Dict[Text, Any] # pylint: disable=invalid-name
except ImportError:
pass
HEADERS = {}
def set_headers(headers):
HEADERS.update(headers)
def load_info(url, params=None, default=None, resp_type = 'json'):
# type: (Text, Optional[Dict[Text, Union[Text, List[Text]]]]) -> Union[dict, list]
"""
Load info from external api
:param url: API endpoint URL
:param params: URL query params
:default: object to return if there is an error
:resp_type: what to return to the calling function
:return: API response or default on error
"""
theerror = ''
if params:
url = url + '?' + urlencode(params)
req = Request(url, headers=HEADERS)
try:
response = urlopen(req)
except URLError as e:
if hasattr(e, 'reason'):
theerror = {'error': 'failed to reach the remote site\nReason: {}'.format(e.reason)}
elif hasattr(e, 'code'):
theerror = {'error': 'remote site unable to fulfill the request\nError code: {}'.format(e.code)}
if default is not None:
return default
else:
return theerror
if resp_type.lower() == 'json':
resp = json.loads(response.read().decode('utf-8'))
else:
resp = response.read().decode('utf-8')
return resp

View File

@ -0,0 +1,87 @@
from . import api_utils
try:
from urllib import quote
except ImportError: # py2 / py3
from urllib.parse import quote
API_KEY = '384afe262ee0962545a752ff340e3ce4'
API_URL = 'https://webservice.fanart.tv/v3/movies/{}'
ARTMAP = {
'movielogo': 'clearlogo',
'hdmovielogo': 'clearlogo',
'hdmovieclearart': 'clearart',
'movieart': 'clearart',
'moviedisc': 'discart',
'moviebanner': 'banner',
'moviethumb': 'landscape',
'moviebackground': 'fanart',
'movieposter': 'poster'
}
def get_details(uniqueids, clientkey, language, set_tmdbid):
media_id = _get_mediaid(uniqueids)
if not media_id:
return {}
movie_data = _get_data(media_id, clientkey)
movieset_data = _get_data(set_tmdbid, clientkey)
if not movie_data and not movieset_data:
return {}
movie_art = {}
movieset_art = {}
if movie_data:
movie_art = _parse_data(movie_data, language)
if movieset_data:
movieset_art = _parse_data(movieset_data, language)
movieset_art = {'set.' + key: value for key, value in movieset_art.items()}
available_art = movie_art
available_art.update(movieset_art)
return {'available_art': available_art}
def _get_mediaid(uniqueids):
for source in ('tmdb', 'imdb', 'unknown'):
if source in uniqueids:
return uniqueids[source]
def _get_data(media_id, clientkey):
headers = {'api-key': API_KEY}
if clientkey:
headers['client-key'] = clientkey
api_utils.set_headers(headers)
fanarttv_url = API_URL.format(media_id)
return api_utils.load_info(fanarttv_url, default={})
def _parse_data(data, language):
result = {}
for arttype, artlist in data.items():
if arttype not in ARTMAP:
continue
for image in artlist:
image_lang = _get_imagelanguage(arttype, image)
if image_lang and image_lang != language:
continue
generaltype = ARTMAP[arttype]
if generaltype == 'poster' and not image_lang:
generaltype = 'keyart'
if artlist and generaltype not in result:
result[generaltype] = []
url = quote(image['url'], safe="%/:=&?~#+!$,;'@()*[]")
resultimage = {'url': url, 'preview': url.replace('.fanart.tv/fanart/', '.fanart.tv/preview/')}
result[generaltype].append(resultimage)
return result
def _get_imagelanguage(arttype, image):
if 'lang' not in image or arttype == 'moviebackground':
return None
if arttype in ('movielogo', 'hdmovielogo', 'hdmovieclearart', 'movieart', 'moviebanner',
'moviethumb', 'moviedisc'):
return image['lang'] if image['lang'] not in ('', '00') else 'en'
# movieposter may or may not have a title and thus need a language
return image['lang'] if image['lang'] not in ('', '00') else None

View File

@ -0,0 +1,72 @@
# -*- coding: UTF-8 -*-
#
# Copyright (C) 2020, Team Kodi
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
# IMDb ratings based on code in metadata.themoviedb.org.python by Team Kodi
# pylint: disable=missing-docstring
import re
from . import api_utils
from . import get_imdb_id
IMDB_RATINGS_URL = 'https://www.imdb.com/title/{}/'
IMDB_RATING_REGEX = re.compile(r'itemprop="ratingValue".*?>.*?([\d.]+).*?<')
IMDB_VOTES_REGEX = re.compile(r'itemprop="ratingCount".*?>.*?([\d,]+).*?<')
IMDB_TOP250_REGEX = re.compile(r'Top Rated Movies #(\d+)')
def get_details(uniqueids):
imdb_id = get_imdb_id(uniqueids)
if not imdb_id:
return {}
votes, rating, top250 = _get_ratinginfo(imdb_id)
return _assemble_imdb_result(votes, rating, top250)
def _get_ratinginfo(imdb_id):
response = api_utils.load_info(IMDB_RATINGS_URL.format(imdb_id), default = '', resp_type='text')
return _parse_imdb_result(response)
def _assemble_imdb_result(votes, rating, top250):
result = {}
if top250:
result['info'] = {'top250': top250}
if votes and rating:
result['ratings'] = {'imdb': {'votes': votes, 'rating': rating}}
return result
def _parse_imdb_result(input_html):
rating = _parse_imdb_rating(input_html)
votes = _parse_imdb_votes(input_html)
top250 = _parse_imdb_top250(input_html)
return votes, rating, top250
def _parse_imdb_rating(input_html):
match = re.search(IMDB_RATING_REGEX, input_html)
if (match):
return float(match.group(1))
return None
def _parse_imdb_votes(input_html):
match = re.search(IMDB_VOTES_REGEX, input_html)
if (match):
return int(match.group(1).replace(',', ''))
return None
def _parse_imdb_top250(input_html):
match = re.search(IMDB_TOP250_REGEX, input_html)
if (match):
return int(match.group(1))
return None

View File

@ -0,0 +1,249 @@
from datetime import datetime, timedelta
from . import tmdbapi
class TMDBMovieScraper(object):
def __init__(self, url_settings, language, certification_country):
self.url_settings = url_settings
self.language = language
self.certification_country = certification_country
self._urls = None
self.tmdbapi = tmdbapi
@property
def urls(self):
if not self._urls:
self._urls = _load_base_urls(self.url_settings)
return self._urls
def search(self, title, year=None):
search_media_id = _parse_media_id(title)
if search_media_id:
if search_media_id['type'] == 'tmdb':
result = _get_movie(search_media_id['id'], self.language, True)
result = [result]
else:
response = tmdbapi.find_movie_by_external_id(search_media_id['id'], language=self.language)
theerror = response.get('error')
if theerror:
return 'error: {}'.format(theerror)
result = response.get('movie_results')
if 'error' in result:
return result
else:
response = tmdbapi.search_movie(query=title, year=year, language=self.language)
theerror = response.get('error')
if theerror:
return 'error: {}'.format(theerror)
result = response['results']
urls = self.urls
def is_best(item):
return item['title'].lower() == title and (
not year or item.get('release_date', '').startswith(year))
if result and not is_best(result[0]):
best_first = next((item for item in result if is_best(item)), None)
if best_first:
result = [best_first] + [item for item in result if item is not best_first]
for item in result:
if item.get('poster_path'):
item['poster_path'] = urls['preview'] + item['poster_path']
if item.get('backdrop_path'):
item['backdrop_path'] = urls['preview'] + item['backdrop_path']
return result
def get_details(self, uniqueids):
media_id = uniqueids.get('tmdb') or uniqueids.get('imdb')
details = self._gather_details(media_id)
if not details:
return None
if details.get('error'):
return details
return self._assemble_details(**details)
def _gather_details(self, media_id):
movie = _get_movie(media_id, self.language)
if not movie or movie.get('error'):
return movie
# Don't specify language to get English text for fallback
movie_fallback = _get_movie(media_id)
collection = _get_moviecollection(movie['belongs_to_collection'].get('id'), self.language) if \
movie['belongs_to_collection'] else None
collection_fallback = _get_moviecollection(movie['belongs_to_collection'].get('id')) if \
movie['belongs_to_collection'] else None
return {'movie': movie, 'movie_fallback': movie_fallback, 'collection': collection,
'collection_fallback': collection_fallback}
def _assemble_details(self, movie, movie_fallback, collection, collection_fallback):
info = {
'title': movie['title'],
'originaltitle': movie['original_title'],
'plot': movie.get('overview') or movie_fallback.get('overview'),
'tagline': movie.get('tagline') or movie_fallback.get('tagline'),
'studio': _get_names(movie['production_companies']),
'genre': _get_names(movie['genres']),
'country': _get_names(movie['production_countries']),
'credits': _get_cast_members(movie['casts'], 'crew', 'Writing', ['Screenplay', 'Writer', 'Author']),
'director': _get_cast_members(movie['casts'], 'crew', 'Directing', ['Director']),
'premiered': movie['release_date'],
'tag': _get_names(movie['keywords']['keywords'])
}
if 'countries' in movie['releases']:
certcountry = self.certification_country.upper()
for country in movie['releases']['countries']:
if country['iso_3166_1'] == certcountry and country['certification']:
info['mpaa'] = country['certification']
break
trailer = _parse_trailer(movie.get('trailers', {}), movie_fallback.get('trailers', {}))
if trailer:
info['trailer'] = trailer
if collection:
info['set'] = collection.get('name') or collection_fallback.get('name')
info['setoverview'] = collection.get('overview') or collection_fallback.get('overview')
if movie.get('runtime'):
info['duration'] = movie['runtime'] * 60
ratings = {'themoviedb': {'rating': float(movie['vote_average']), 'votes': int(movie['vote_count'])}}
uniqueids = {'tmdb': movie['id'], 'imdb': movie['imdb_id']}
cast = [{
'name': actor['name'],
'role': actor['character'],
'thumbnail': self.urls['original'] + actor['profile_path']
if actor['profile_path'] else "",
'order': actor['order']
}
for actor in movie['casts'].get('cast', [])
]
available_art = _parse_artwork(movie, collection, self.urls, self.language)
_info = {'set_tmdbid': movie['belongs_to_collection'].get('id')
if movie['belongs_to_collection'] else None}
return {'info': info, 'ratings': ratings, 'uniqueids': uniqueids, 'cast': cast,
'available_art': available_art, '_info': _info}
def _parse_media_id(title):
if title.startswith('tt') and title[2:].isdigit():
return {'type': 'imdb', 'id':title} # IMDB ID works alone because it is clear
title = title.lower()
if title.startswith('tmdb/') and title[5:].isdigit(): # TMDB ID
return {'type': 'tmdb', 'id':title[5:]}
elif title.startswith('imdb/tt') and title[7:].isdigit(): # IMDB ID with prefix to match
return {'type': 'imdb', 'id':title[5:]}
return None
def _get_movie(mid, language=None, search=False):
details = None if search else \
'trailers,images,releases,casts,keywords' if language is not None else \
'trailers'
response = tmdbapi.get_movie(mid, language=language, append_to_response=details)
theerror = response.get('error')
if theerror:
return 'error: {}'.format(theerror)
else:
return response
def _get_moviecollection(collection_id, language=None):
if not collection_id:
return None
details = 'images'
response = tmdbapi.get_collection(collection_id, language=language, append_to_response=details)
theerror = response.get('error')
if theerror:
return 'error: {}'.format(theerror)
else:
return response
def _parse_artwork(movie, collection, urlbases, language):
if language:
# Image languages don't have regional variants
language = language.split('-')[0]
posters = []
landscape = []
fanart = []
if 'images' in movie:
posters = _get_images_with_fallback(movie['images']['posters'], urlbases, language)
landscape = _get_images(movie['images']['backdrops'], urlbases, language)
fanart = _get_images(movie['images']['backdrops'], urlbases, None)
setposters = []
setlandscape = []
setfanart = []
if collection and 'images' in collection:
setposters = _get_images_with_fallback(collection['images']['posters'], urlbases, language)
setlandscape = _get_images(collection['images']['backdrops'], urlbases, language)
setfanart = _get_images(collection['images']['backdrops'], urlbases, None)
return {'poster': posters, 'landscape': landscape, 'fanart': fanart,
'set.poster': setposters, 'set.landscape': setlandscape, 'set.fanart': setfanart}
def _get_images_with_fallback(imagelist, urlbases, language, language_fallback='en'):
images = _get_images(imagelist, urlbases, language)
# Add backup images
if language != language_fallback:
images.extend(_get_images(imagelist, urlbases, language_fallback))
# Add any images if nothing set so far
if not images:
images = _get_images(imagelist, urlbases)
return images
def _get_images(imagelist, urlbases, language='_any'):
result = []
for img in imagelist:
if language != '_any' and img['iso_639_1'] != language:
continue
result.append({
'url': urlbases['original'] + img['file_path'],
'preview': urlbases['preview'] + img['file_path'],
})
return result
def _get_date_numeric(datetime_):
return (datetime_ - datetime(1970, 1, 1)).total_seconds()
def _load_base_urls(url_settings):
urls = {}
# urls['original'] = url_settings.getSettingString('originalUrl')
# urls['preview'] = url_settings.getSettingString('previewUrl')
# last_updated = url_settings.getSettingString('lastUpdated')
urls['original'] = ""
urls['preview'] = ""
last_updated = "0"
if not urls['original'] or not urls['preview'] or not last_updated or \
float(last_updated) < _get_date_numeric(datetime.now() - timedelta(days=30)):
conf = tmdbapi.get_configuration()
if conf:
urls['original'] = conf['images']['secure_base_url'] + 'original'
urls['preview'] = conf['images']['secure_base_url'] + 'w780'
# url_settings.setSetting('originalUrl', urls['original'])
# url_settings.setSetting('previewUrl', urls['preview'])
# url_settings.setSetting('lastUpdated', str(_get_date_numeric(datetime.now())))
return urls
def _parse_trailer(trailers, fallback):
if trailers.get('youtube'):
return 'plugin://plugin.video.youtube/?action=play_video&videoid='+trailers['youtube'][0]['source']
if fallback.get('youtube'):
return 'plugin://plugin.video.youtube/?action=play_video&videoid='+fallback['youtube'][0]['source']
return None
def _get_names(items):
return [item['name'] for item in items] if items else []
def _get_cast_members(casts, casttype, department, jobs):
result = []
if casttype in casts:
for cast in casts[casttype]:
if cast['department'] == department and cast['job'] in jobs and cast['name'] not in result:
result.append(cast['name'])
return result

View File

@ -0,0 +1,129 @@
# -*- coding: UTF-8 -*-
#
# Copyright (C) 2020, Team Kodi
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
# pylint: disable=missing-docstring
"""Functions to interact with TMDb API."""
from . import api_utils
try:
from typing import Optional, Text, Dict, List, Any # pylint: disable=unused-import
InfoType = Dict[Text, Any] # pylint: disable=invalid-name
except ImportError:
pass
HEADERS = (
('User-Agent', 'Kodi Movie scraper by Team Kodi'),
('Accept', 'application/json'),
)
api_utils.set_headers(dict(HEADERS))
TMDB_PARAMS = {'api_key': 'f090bb54758cabf231fb605d3e3e0468'}
BASE_URL = 'https://api.themoviedb.org/3/{}'
SEARCH_URL = BASE_URL.format('search/movie')
FIND_URL = BASE_URL.format('find/{}')
MOVIE_URL = BASE_URL.format('movie/{}')
COLLECTION_URL = BASE_URL.format('collection/{}')
CONFIG_URL = BASE_URL.format('configuration')
def search_movie(query, year=None, language=None):
# type: (Text) -> List[InfoType]
"""
Search for a movie
:param title: movie title to search
:param year: the year to search (optional)
:param language: the language filter for TMDb (optional)
:return: a list with found movies
"""
theurl = SEARCH_URL
params = _set_params(None, language)
params['query'] = query
if year is not None:
params['year'] = str(year)
return api_utils.load_info(theurl, params=params)
def find_movie_by_external_id(external_id, language=None):
# type: (Text) -> List[InfoType]
"""
Find movie based on external ID
:param mid: external ID
:param language: the language filter for TMDb (optional)
:return: the movie or error
"""
theurl = FIND_URL.format(external_id)
params = _set_params(None, language)
params['external_source'] = 'imdb_id'
return api_utils.load_info(theurl, params=params)
def get_movie(mid, language=None, append_to_response=None):
# type: (Text) -> List[InfoType]
"""
Get movie details
:param mid: TMDb movie ID
:param language: the language filter for TMDb (optional)
:append_to_response: the additional data to get from TMDb (optional)
:return: the movie or error
"""
try:
theurl = MOVIE_URL.format(mid)
return api_utils.load_info(theurl, params=_set_params(append_to_response, language))
except Exception as e:
print(repr(e))
def get_collection(collection_id, language=None, append_to_response=None):
# type: (Text) -> List[InfoType]
"""
Get movie collection information
:param collection_id: TMDb collection ID
:param language: the language filter for TMDb (optional)
:append_to_response: the additional data to get from TMDb (optional)
:return: the movie or error
"""
theurl = COLLECTION_URL.format(collection_id)
return api_utils.load_info(theurl, params=_set_params(append_to_response, language))
def get_configuration():
# type: (Text) -> List[InfoType]
"""
Get configuration information
:return: configuration details or error
"""
return api_utils.load_info(CONFIG_URL, params=TMDB_PARAMS.copy())
def _set_params(append_to_response, language):
params = TMDB_PARAMS.copy()
img_lang = 'en,null'
if language is not None:
params['language'] = language
img_lang = '%s,en,null' % language[0:2]
if append_to_response is not None:
params['append_to_response'] = append_to_response
if 'images' in append_to_response:
params['include_image_language'] = img_lang
return params

View File

@ -0,0 +1,55 @@
# -*- coding: UTF-8 -*-
#
# Copyright (C) 2020, Team Kodi
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
# pylint: disable=missing-docstring
"""Functions to interact with Trakt API."""
from __future__ import absolute_import, unicode_literals
from . import api_utils
from . import get_imdb_id
try:
from typing import Optional, Text, Dict, List, Any # pylint: disable=unused-import
InfoType = Dict[Text, Any] # pylint: disable=invalid-name
except ImportError:
pass
HEADERS = (
('User-Agent', 'Kodi Movie scraper by Team Kodi'),
('Accept', 'application/json'),
('trakt-api-key', '5f2dc73b6b11c2ac212f5d8b4ec8f3dc4b727bb3f026cd254d89eda997fe64ae'),
('trakt-api-version', '2'),
('Content-Type', 'application/json'),
)
api_utils.set_headers(dict(HEADERS))
MOVIE_URL = 'https://api.trakt.tv/movies/{}'
def get_trakt_ratinginfo(uniqueids):
imdb_id = get_imdb_id(uniqueids)
result = {}
url = MOVIE_URL.format(imdb_id)
params = {'extended': 'full'}
movie_info = api_utils.load_info(url, params=params, default={})
if(movie_info):
if 'votes' in movie_info and 'rating' in movie_info:
result['ratings'] = {'trakt': {'votes': int(movie_info['votes']), 'rating': float(movie_info['rating'])}}
elif 'rating' in movie_info:
result['ratings'] = {'trakt': {'rating': float(movie_info['rating'])}}
return result

Binary file not shown.