From d8272758ff38bb09e1a7c5987012abd9f76a324c Mon Sep 17 00:00:00 2001 From: Gaisberg Date: Fri, 22 Dec 2023 00:09:59 +0200 Subject: [PATCH] Address slow loading times and cpu usage, hacky but what the hell --- backend/program/content/__init__.py | 1 + backend/program/content/mdblist.py | 17 +++--- backend/program/content/overseerr.py | 17 +++--- backend/program/debrid/realdebrid.py | 5 +- backend/program/libraries/plex.py | 87 ++++++++++++--------------- backend/program/scrapers/__init__.py | 2 + backend/program/scrapers/torrentio.py | 1 + backend/program/symlink.py | 2 + backend/program/updaters/trakt.py | 4 +- backend/utils/utils.py | 21 ++++--- 10 files changed, 77 insertions(+), 80 deletions(-) diff --git a/backend/program/content/__init__.py b/backend/program/content/__init__.py index 112144af..cbfda43d 100644 --- a/backend/program/content/__init__.py +++ b/backend/program/content/__init__.py @@ -35,6 +35,7 @@ def run(self) -> None: for service in self.services: if service.initialized: service.run() + time.sleep(1) def start(self) -> None: self.running = True diff --git a/backend/program/content/mdblist.py b/backend/program/content/mdblist.py index b1be9c09..2d64f8c8 100644 --- a/backend/program/content/mdblist.py +++ b/backend/program/content/mdblist.py @@ -21,7 +21,6 @@ def __init__(self, media_items: MediaItemContainer): self.requests_per_2_minutes = self._calculate_request_time() self.rate_limiter = RateLimiter(self.requests_per_2_minutes, 120, True) self.initialized = True - self.last_items = [] def _validate_settings(self): response = ping( @@ -40,15 +39,13 @@ def run(self): items += self._get_items_from_list( list_id, self.settings["api_key"] ) - if len(items) != len(self.media_items): - self.last_items = items - new_items = [item for item in items if item not in self.media_items] - container = self.updater.create_items(new_items) - for item in container: - item.set("requested_by", "Mdblist") - added_items = self.media_items.extend(container) - if len(added_items) > 0: - logger.info("Added %s items", len(added_items)) + new_items = [item for item in items if item not in self.media_items] + container = self.updater.create_items(new_items) + for item in container: + item.set("requested_by", "Mdblist") + added_items = self.media_items.extend(container) + if len(added_items) > 0: + logger.info("Added %s items", len(added_items)) except RateLimitExceeded: pass diff --git a/backend/program/content/overseerr.py b/backend/program/content/overseerr.py index 0c164854..5d647fab 100644 --- a/backend/program/content/overseerr.py +++ b/backend/program/content/overseerr.py @@ -19,7 +19,6 @@ def __init__(self, media_items: MediaItemContainer): return self.updater = Trakt() self.not_found_ids = [] - self.last_items = [] self.initialized = True def _validate_settings(self): @@ -37,15 +36,13 @@ def run(self): """Fetch media from overseerr and add them to media_items attribute if they are not already there""" items = self._get_items_from_overseerr(10000) - if len(items) != len(self.last_items): - self.last_items = items - new_items = [item for item in items if item not in self.media_items] - container = self.updater.create_items(new_items) - for item in container: - item.set("requested_by", "Overseerr") - added_items = self.media_items.extend(container) - if len(added_items) > 0: - logger.info("Added %s items", len(added_items)) + new_items = [item for item in items if item not in self.media_items] + container = self.updater.create_items(new_items) + for item in container: + item.set("requested_by", "Overseerr") + added_items = self.media_items.extend(container) + if len(added_items) > 0: + logger.info("Added %s items", len(added_items)) def _get_items_from_overseerr(self, amount: int): """Fetch media from overseerr""" diff --git a/backend/program/debrid/realdebrid.py b/backend/program/debrid/realdebrid.py index efde9308..84255848 100644 --- a/backend/program/debrid/realdebrid.py +++ b/backend/program/debrid/realdebrid.py @@ -59,6 +59,7 @@ def _validate_settings(self): def run(self): while self.running: self.download() + time.sleep(1) def start(self) -> None: self.running = True @@ -131,7 +132,9 @@ def _download_item(self, item): if item.type == "season": log_string = f"{item.parent.title} S{item.number}" if item.type == "episode": - log_string = f"{item.parent.parent.title} S{item.parent.number}E{item.number}" + log_string = ( + f"{item.parent.parent.title} S{item.parent.number}E{item.number}" + ) logger.debug("Downloaded %s", log_string) return 1 diff --git a/backend/program/libraries/plex.py b/backend/program/libraries/plex.py index 7c183177..f10beea4 100644 --- a/backend/program/libraries/plex.py +++ b/backend/program/libraries/plex.py @@ -59,6 +59,7 @@ def run(self): while self.running: self._update_sections() self._update_items() + time.sleep(1) def start(self): self.running = True @@ -99,10 +100,8 @@ def _update_items(self): continue processed_sections.add(section.key) + matched_items = self.match_items(items) - self.media_items.extend(items) - - matched_items = self.match_items(items, self.media_items) if matched_items > 0: logger.info(f"Found {matched_items} new items") @@ -143,62 +142,43 @@ def _update_sections(self): episode.set("update_folder", "updated") log_string = f"{item.title} season {season.number} episode {episode.number}" break - if log_string: - logger.debug("Updated section %s for %s", section.title, log_string) + if log_string: + logger.debug("Updated section %s for %s", section.title, log_string) def _create_item(self, item): - new_item = _map_item_from_data(item, item.type) + new_item = _map_item_from_data(item) if new_item and item.type == "show": for season in item.seasons(): if season.seasonNumber != 0: - new_season = _map_item_from_data(season, "season") + new_season = _map_item_from_data(season) if new_season: new_season_episodes = [] for episode in season.episodes(): - new_episode = _map_item_from_data(episode, "episode") + new_episode = _map_item_from_data(episode) if new_episode: new_season_episodes.append(new_episode) new_season.episodes = new_season_episodes new_item.seasons.append(new_season) return new_item - def match_items(self, found_items: List[MediaItem], media_items: List[MediaItem]): + def match_items(self, found_items: List[MediaItem]): """Matches items in given mediacontainer that are not in library to items that are in library""" items_to_update = 0 - for item in media_items: - if item.state != MediaItemState.LIBRARY: - if item.type == "movie": - for found_item in found_items: - if ( - found_item.type == "movie" - and found_item.imdb_id == item.imdb_id - ): - self._update_item(item, found_item) - items_to_update += 1 - break - if item.type == "show": - for found_item in found_items: - if found_item.type == "show": - for found_season in found_item.seasons: - for found_episode in found_season.episodes: - for season in item.seasons: - for episode in season.episodes: - if ( - episode.state - is not MediaItemState.LIBRARY - ): - if ( - episode.imdb_id - == found_episode.imdb_id - ): - self._update_item( - episode, found_episode - ) - items_to_update += 1 - break - + for item in self.media_items: + if item.state not in [ + MediaItemState.LIBRARY, + MediaItemState.LIBRARY_PARTIAL, + ]: + for found_item in found_items: + if found_item.imdb_id == item.imdb_id: + self._update_item(item, found_item) + items_to_update += 1 + break + # Leaving this here as a reminder to not forget about deleting items that are removed from plex, needs to be revisited + # if item.state is MediaItemState.LIBRARY and item not in found_items: + # self.media_items.remove(item) return items_to_update def _update_item(self, item: MediaItem, library_item: MediaItem): @@ -207,16 +187,27 @@ def _update_item(self, item: MediaItem, library_item: MediaItem): items found""" item.set("guid", library_item.guid) item.set("key", library_item.key) + if item.type == "show": + for season in item.seasons: + for episode in season.episodes: + for found_season in library_item.seasons: + if found_season.number == season.number: + for found_episode in found_season.episodes: + if found_episode.number == episode.number: + episode.set("guid", found_episode.guid) + episode.set("key", found_episode.key) + break + break def _is_wanted_section(self, section): return any(self.library_path in location for location in section.locations) -def _map_item_from_data(item, item_type): +def _map_item_from_data(item): """Map Plex API data to MediaItemContainer.""" file = None guid = getattr(item, "guid", None) - if item_type in ["movie", "episode"]: + if item.type in ["movie", "episode"]: file = getattr(item, "locations", [None])[0].split("/")[-1] genres = [genre.tag for genre in getattr(item, "genres", [])] title = getattr(item, "title", None) @@ -227,7 +218,7 @@ def _map_item_from_data(item, item_type): imdb_id = None aired_at = None - if item_type != "season": + if item.type in ["movie", "show"]: guids = getattr(item, "guids", []) imdb_id = next( (guid.id.split("://")[-1] for guid in guids if "imdb" in guid.id), None @@ -246,14 +237,14 @@ def _map_item_from_data(item, item_type): } # Instantiate the appropriate subclass based on 'item_type' - if item_type == "movie": + if item.type == "movie": return Movie(media_item_data) - elif item_type == "show": + elif item.type == "show": return Show(media_item_data) - elif item_type == "season": + elif item.type == "season": media_item_data["number"] = season_number return Season(media_item_data) - elif item_type == "episode": + elif item.type == "episode": media_item_data["number"] = episode_number media_item_data["season_number"] = season_number return Episode(media_item_data) diff --git a/backend/program/scrapers/__init__.py b/backend/program/scrapers/__init__.py index 40015195..491b8643 100644 --- a/backend/program/scrapers/__init__.py +++ b/backend/program/scrapers/__init__.py @@ -26,6 +26,8 @@ def run(self) -> None: for service in self.services: if service.initialized: service.run() + time.sleep(1) + def start(self) -> None: self.running = True diff --git a/backend/program/scrapers/torrentio.py b/backend/program/scrapers/torrentio.py index af5bb9da..c956ed1a 100644 --- a/backend/program/scrapers/torrentio.py +++ b/backend/program/scrapers/torrentio.py @@ -63,6 +63,7 @@ def _scrape_items(self, items: list): amount_scraped = 0 for item in items: data = self.api_scrape(item) + log_string = item.title if item.type == "season": log_string = f"{item.parent.title} S{item.number}" if item.type == "episode": diff --git a/backend/program/symlink.py b/backend/program/symlink.py index ca0e499e..b1e21ec6 100644 --- a/backend/program/symlink.py +++ b/backend/program/symlink.py @@ -49,6 +49,8 @@ def __init__(self, media_items: MediaItemContainer): def run(self): while self.running: self._run() + time.sleep(1) + def start(self): self.running = True diff --git a/backend/program/updaters/trakt.py b/backend/program/updaters/trakt.py index 9a6d48b2..de21b6fa 100644 --- a/backend/program/updaters/trakt.py +++ b/backend/program/updaters/trakt.py @@ -1,8 +1,8 @@ """Trakt updater module""" from datetime import datetime from os import path -from backend.utils.logger import get_data_path, logger -from backend.utils.request import get +from utils.logger import get_data_path, logger +from utils.request import get from program.media import ( Episode, MediaItemContainer, diff --git a/backend/utils/utils.py b/backend/utils/utils.py index b7d186e5..9b6e0a22 100644 --- a/backend/utils/utils.py +++ b/backend/utils/utils.py @@ -35,22 +35,21 @@ def run(self): time.sleep(10) - -class Parser(): +class Parser: def __init__(self): self.resolution = ["1080p", "720p"] self.language = ["English"] def _parse(self, string): parse = PTN.parse(string) - + # episodes episodes = [] if parse.get("episode", False): episode = parse.get("episode") if type(episode) == list: - for sub_episode in episode: - episodes.append(int(sub_episode)) + for sub_episode in episode: + episodes.append(int(sub_episode)) else: episodes.append(int(episode)) @@ -60,20 +59,24 @@ def _parse(self, string): if not language: language = "English" extended = parse.get("extended") - + return { "episodes": episodes or [], "resolution": resolution or [], "language": language or [], "extended": extended, } - + def episodes(self, string): parse = self._parse(string) return parse["episodes"] def parse(self, string): parse = self._parse(string) - return parse["resolution"] in self.resolution and parse["language"] in self.language + return ( + parse["resolution"] in self.resolution + and parse["language"] in self.language + ) + -parser = Parser() \ No newline at end of file +parser = Parser()