Merge branch 'development'

README.md

@@ -3,12 +3,9 @@ Bazarr is a companion application to Sonarr and Radarr. It manages and downloads
 
 Be aware that Bazarr doesn't scan disk to detect series and movies: It only takes care of the series and movies that are indexed in Sonarr and Radarr.
 
-## Support on Beerpay
+## Support on Paypal
 At the request of some, here is a way to demonstrate your appreciation for the efforts made in the development of Bazarr:
-[![Beerpay](https://beerpay.io/morpheus65535/bazarr/badge.svg?style=beer-square)](https://beerpay.io/morpheus65535/bazarr)
-
-You can also make a wish but keep in mind that we do not commit to make it happen:
-[![Beerpay](https://beerpay.io/morpheus65535/bazarr/make-wish.svg?style=flat-square)](https://beerpay.io/morpheus65535/bazarr?focus=wish)
+[![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=XHHRWXT9YB7WE&source=url)
 
 # Status
 [![GitHub issues](https://img.shields.io/github/issues/morpheus65535/bazarr.svg?style=flat-square)](https://github.com/morpheus65535/bazarr/issues)
@@ -69,6 +66,7 @@ If you need something that is not already part of Bazarr, feel free to create a
 * Supersubtitles
 * Titlovi
 * TVSubtitles
+* Wizdom
 * XSubs
 * Zimuku
 

bazarr.py

@@ -14,7 +14,7 @@
 def check_python_version():
     python_version = platform.python_version_tuple()
     minimum_py2_tuple = (2, 7, 13)
-    minimum_py3_tuple = (3, 6, 0)
+    minimum_py3_tuple = (3, 7, 0)
     minimum_py2_str = ".".join(str(i) for i in minimum_py2_tuple)
     minimum_py3_str = ".".join(str(i) for i in minimum_py3_tuple)
 

bazarr/check_update.py

@@ -60,7 +60,7 @@ def check_and_apply_update():
         if releases is None:
             notifications.write(msg='Could not get releases from GitHub.',
                                 queue='check_update', type='warning')
-            logging.warn('BAZARR Could not get releases from GitHub.')
+            logging.warning('BAZARR Could not get releases from GitHub.')
             return
         else:
             release = releases[0]

bazarr/embedded_subs_reader.py

@@ -3,11 +3,9 @@
 from enzyme.exceptions import MalformedMKVError
 import logging
 import os
-import subprocess
-import locale
+from knowit import api
 
 from utils import get_binary
-from pyprobe.pyprobe import VideoFileParser
 
 class NotMKVAndNoFFprobe(Exception):
     pass
@@ -23,11 +21,18 @@ def list_languages(self, file):
         subtitles_list = []
 
         if self.ffprobe:
-            parser = VideoFileParser(ffprobe=self.ffprobe, includeMissing=True, rawMode=False)
-            data = parser.parseFfprobe(file)
-
-            for detected_language in data['subtitles']:
-                subtitles_list.append([detected_language['language'], detected_language['forced'], detected_language["codec"]])
+            api.initialize({'provider': 'ffmpeg', 'ffmpeg': self.ffprobe})
+            data = api.know(file)
+
+            if 'subtitle' in data:
+                for detected_language in data['subtitle']:
+                    if 'language' in detected_language:
+                        language = detected_language['language'].alpha3
+                        forced = detected_language['forced'] if 'forced' in detected_language else None
+                        codec = detected_language['format'] if 'format' in detected_language else None
+                        subtitles_list.append([language, forced, codec])
+                    else:
+                        continue
         else:
             if os.path.splitext(file)[1] == '.mkv':
                 with open(file, 'rb') as f:

bazarr/get_languages.py

@@ -3,6 +3,7 @@
 from __future__ import absolute_import
 import os
 import pycountry
+import ast
 
 from get_args import args
 from subzero.language import Language
@@ -76,5 +77,48 @@ def get_language_set():
     return language_set
 
 
+def clean_desired_languages():
+    from list_subtitles import list_missing_subtitles, list_missing_subtitles_movies
+    enabled_languages = []
+    enabled_languages_temp = database.execute("SELECT code2 FROM table_settings_languages WHERE enabled=1")
+    for language in enabled_languages_temp:
+        enabled_languages.append(language['code2'])
+
+    series_languages = database.execute("SELECT sonarrSeriesId, languages FROM table_shows")
+    movies_languages = database.execute("SELECT radarrId, languages FROM table_movies")
+
+    for item in series_languages:
+        if item['languages'] != 'None':
+            try:
+                languages_list = ast.literal_eval(item['languages'])
+            except:
+                pass
+            else:
+                cleaned_languages_list = []
+                for language in languages_list:
+                    if language in enabled_languages:
+                        cleaned_languages_list.append(language)
+                if cleaned_languages_list != languages_list:
+                    database.execute("UPDATE table_shows SET languages=? WHERE sonarrSeriesId=?",
+                                     (str(cleaned_languages_list), item['sonarrSeriesId']))
+                    list_missing_subtitles(no=item['sonarrSeriesId'])
+
+    for item in movies_languages:
+        if item['languages'] != 'None':
+            try:
+                languages_list = ast.literal_eval(item['languages'])
+            except:
+                pass
+            else:
+                cleaned_languages_list = []
+                for language in languages_list:
+                    if language in enabled_languages:
+                        cleaned_languages_list.append(language)
+                if cleaned_languages_list != languages_list:
+                    database.execute("UPDATE table_movies SET languages=? WHERE radarrId=?",
+                                     (str(cleaned_languages_list), item['radarrId']))
+                    list_missing_subtitles_movies(no=item['radarrId'])
+
+
 if __name__ == '__main__':
     load_language_in_db()

bazarr/get_movies.py

@@ -25,6 +25,8 @@ def update_all_movies():
 def update_movies():
     logging.debug('BAZARR Starting movie sync from Radarr.')
     apikey_radarr = settings.radarr.apikey
+
+    radarr_version = get_radarr_version()
     movie_default_enabled = settings.general.getboolean('movie_default_enabled')
     movie_default_language = settings.general.movie_default_language
     movie_default_hi = settings.general.movie_default_hi
@@ -130,6 +132,8 @@ def update_movies():
                                 videoCodec = None
                                 audioCodec = None
 
+                            audio_language = profile_id_to_language(movie['qualityProfileId'], audio_profiles)
+
                             # Add movies in radarr to current movies list
                             current_movies_radarr.append(six.text_type(movie['tmdbId']))
 
@@ -140,7 +144,7 @@ def update_movies():
                                                          'tmdbId': six.text_type(movie["tmdbId"]),
                                                          'poster': poster,
                                                          'fanart': fanart,
-                                                         'audio_language': profile_id_to_language(movie['qualityProfileId'], audio_profiles),
+                                                         'audio_language': audio_language,
                                                          'sceneName': sceneName,
                                                          'monitored': six.text_type(bool(movie['monitored'])),
                                                          'year': six.text_type(movie['year']),
@@ -165,7 +169,7 @@ def update_movies():
                                                           'overview': overview,
                                                           'poster': poster,
                                                           'fanart': fanart,
-                                                          'audio_language': profile_id_to_language(movie['qualityProfileId'], audio_profiles),
+                                                          'audio_language': audio_language,
                                                           'sceneName': sceneName,
                                                           'monitored': six.text_type(bool(movie['monitored'])),
                                                           'sortTitle': movie['sortTitle'],
@@ -189,7 +193,7 @@ def update_movies():
                                                           'overview': overview,
                                                           'poster': poster,
                                                           'fanart': fanart,
-                                                          'audio_language': profile_id_to_language(movie['qualityProfileId'], audio_profiles),
+                                                          'audio_language': audio_language,
                                                           'sceneName': sceneName,
                                                           'monitored': six.text_type(bool(movie['monitored'])),
                                                           'sortTitle': movie['sortTitle'],
@@ -227,8 +231,8 @@ def update_movies():
 
             for updated_movie in movies_to_update_list:
                 query = dict_converter.convert(updated_movie)
-                database.execute('''UPDATE table_movies SET ''' + query.keys_update + ''' WHERE radarrId = ?''',
-                                 query.values + (updated_movie['radarrId'],))
+                database.execute('''UPDATE table_movies SET ''' + query.keys_update + ''' WHERE tmdbId = ?''',
+                                 query.values + (updated_movie['tmdbId'],))
                 altered_movies.append([updated_movie['tmdbId'],
                                        updated_movie['path'],
                                        updated_movie['radarrId'],
@@ -275,8 +279,11 @@ def get_profile_list():
     radarr_version = get_radarr_version()
     profiles_list = []
     # Get profiles data from radarr
+    if radarr_version.startswith('0'):
+        url_radarr_api_movies = url_radarr() + "/api/profile?apikey=" + apikey_radarr
+    else:
+        url_radarr_api_movies = url_radarr() + "/api/v3/qualityprofile?apikey=" + apikey_radarr
 
-    url_radarr_api_movies = url_radarr() + "/api/profile?apikey=" + apikey_radarr
     try:
         profiles_json = requests.get(url_radarr_api_movies, timeout=60, verify=False)
     except requests.exceptions.ConnectionError as errc:
@@ -290,7 +297,7 @@ def get_profile_list():
         if radarr_version.startswith('0'):
             for profile in profiles_json.json():
                 profiles_list.append([profile['id'], profile['language'].capitalize()])
-        elif radarr_version.startswith('2'):
+        else:
             for profile in profiles_json.json():
                 profiles_list.append([profile['id'], profile['language']['name'].capitalize()])
 

bazarr/get_subtitle.py

@@ -34,14 +34,15 @@
 from get_providers import get_providers, get_providers_auth, provider_throttle, provider_pool
 from get_args import args
 from queueconfig import notifications
-from pyprobe.pyprobe import VideoFileParser
+from knowit import api
 from database import database, dict_mapper
 
 from analytics import track_event
 import six
 from six.moves import range
 from functools import reduce
 from locale import getpreferredencoding
+import chardet
 
 
 def get_video(path, title, sceneName, use_scenename, providers=None, media_type="movie"):
@@ -346,10 +347,20 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa
                 not_matched = scores - matches
                 s.score = score
 
-                releases = ['n/a']
+                releases = []
                 if hasattr(s, 'release_info'):
                     if s.release_info is not None:
-                        releases = s.release_info.split(',')
+                        for s_item in s.release_info.split(','):
+                            if s_item.strip():
+                                releases.append(s_item)
+
+                if len(releases) == 0:
+                    releases = ['n/a']
+
+                if s.uploader and s.uploader.strip():
+                    s_uploader = s.uploader.strip()
+                else:
+                    s_uploader = 'n/a'
 
                 subtitles_list.append(
                     dict(score=round((score / max_score * 100), 2),
@@ -359,7 +370,7 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa
                          provider=s.provider_name,
                          subtitle=codecs.encode(pickle.dumps(s.make_picklable()), "base64").decode(),
                          url=s.page_link, matches=list(matches), dont_matches=list(not_matched),
-                         release_info=releases))
+                         release_info=releases, uploader=s_uploader))
 
             final_subtitles = sorted(subtitles_list, key=lambda x: (x['orig_score'], x['score_without_hash']),
                                      reverse=True)
@@ -470,6 +481,10 @@ def manual_upload_subtitle(path, language, forced, title, scene_name, media_type
     chmod = int(settings.general.chmod, 8) if not sys.platform.startswith(
         'win') and settings.general.getboolean('chmod_enabled') else None
 
+    dest_directory = get_target_folder(path)
+    fake_video_path = None
+    if dest_directory:
+        fake_video_path = os.path.join(dest_directory, os.path.split(path)[1])
     _, ext = os.path.splitext(subtitle.filename)
 
     language = alpha3_from_alpha2(language)
@@ -482,7 +497,7 @@ def manual_upload_subtitle(path, language, forced, title, scene_name, media_type
     if forced:
         lang_obj = Language.rebuild(lang_obj, forced=True)
 
-    subtitle_path = get_subtitle_path(video_path=force_unicode(path),
+    subtitle_path = get_subtitle_path(video_path=force_unicode(fake_video_path if fake_video_path else path),
                                       language=None if single else lang_obj,
                                       extension=ext,
                                       forced_tag=forced)
@@ -492,7 +507,34 @@ def manual_upload_subtitle(path, language, forced, title, scene_name, media_type
     if os.path.exists(subtitle_path):
         os.remove(subtitle_path)
 
-    subtitle.save(subtitle_path)
+    if settings.general.utf8_encode:
+        try:
+            os.remove(subtitle_path + ".tmp")
+        except:
+            pass
+
+        subtitle.save(subtitle_path + ".tmp")
+
+        with open(subtitle_path + ".tmp", 'rb') as fr:
+            text = fr.read()
+
+        try:
+            guess = chardet.detect(text)
+            text = text.decode(guess["encoding"])
+            text = text.encode('utf-8')
+        except UnicodeError:
+            logging.exception("BAZARR subtitles file doesn't seems to be text based. Skipping this file: " +
+                              subtitle_path)
+        else:
+            with open(subtitle_path, 'wb') as fw:
+                fw.write(text)
+        finally:
+            try:
+                os.remove(subtitle_path + ".tmp")
+            except:
+                pass
+    else:
+        subtitle.save(subtitle_path)
 
     if chmod:
         os.chmod(subtitle_path, chmod)
@@ -868,7 +910,7 @@ def refine_from_db(path, video):
                                 "WHERE table_episodes.path = ?", (path_replace_reverse(path),), only_one=True)
 
         if data:
-            video.series, year, country = series_re.match(data['seriesTitle']).groups()
+            video.series = data['seriesTitle']
             video.season = int(data['season'])
             video.episode = int(data['episode'])
             video.title = data['episodeTitle']
@@ -915,37 +957,30 @@ def refine_from_ffprobe(path, video):
     else:
         logging.debug('BAZARR FFprobe used is %s', exe)
 
-    parser = VideoFileParser(ffprobe=exe, includeMissing=True, rawMode=False)
-    data = parser.parseFfprobe(path)
+    api.initialize({'provider': 'ffmpeg', 'ffmpeg': exe})
+    data = api.know(path)
 
     logging.debug('FFprobe found: %s', data)
 
-    if 'videos' not in data:
+    if 'video' not in data:
         logging.debug('BAZARR FFprobe was unable to find video tracks in the file!')
     else:
-        if 'resolution' in data['videos'][0]:
+        if 'resolution' in data['video'][0]:
             if not video.resolution:
-                if data['videos'][0]['resolution'][0] >= 3200:
-                    video.resolution = "2160p"
-                elif data['videos'][0]['resolution'][0] >= 1800:
-                    video.resolution = "1080p"
-                elif data['videos'][0]['resolution'][0] >= 1200:
-                    video.resolution = "720p"
-                elif data['videos'][0]['resolution'][0] >= 0:
-                    video.resolution = "480p"
-        if 'codec' in data['videos'][0]:
+                video.resolution = data['video'][0]['resolution']
+        if 'codec' in data['video'][0]:
             if not video.video_codec:
-                video.video_codec = data['videos'][0]['codec']
-        if 'framerate' in data['videos'][0]:
+                video.video_codec = data['video'][0]['codec']
+        if 'frame_rate' in data['video'][0]:
             if not video.fps:
-                video.fps = data['videos'][0]['framerate']
+                video.fps = data['video'][0]['frame_rate']
 
-    if 'audios' not in data:
+    if 'audio' not in data:
         logging.debug('BAZARR FFprobe was unable to find audio tracks in the file!')
     else:
-        if 'codec' in data['audios'][0]:
+        if 'codec' in data['audio'][0]:
             if not video.audio_codec:
-                video.audio_codec = data['audios'][0]['codec'].upper()
+                video.audio_codec = data['audio'][0]['codec']
 
 
 def upgrade_subtitles():