Release 0.7.5 (#478)

* feat: added log to show total processing scraped results

* Add Zilean scraper: untested - Spoked can test :D (#473)

Zilean is a service that allows you to search for DebridMediaManager sourced arr-less content. When the service is started, it will automatically download and index all the DMM shared hashlists and index them using Lucene. The service provides a search endpoint that allows you to search for content using a query string, and returns a list of filenames and infohashes. There is no clean filtering applied to the search results, the idea behind this endpoint is Riven performs that using RTN. The DMM import reruns on missing pages every hour.
Its written exclusively for Riven
/~https://github.com/iPromKnight/zilean

* feat: Zilean support

* Update to include get_top_title() as well as Show (#475)

* Update zilean.py

Allow shows in zilean

* Update zilean.py

change title to get_top_title()

* feat: update with new changes

* feat: update with new changes

* Jacket Fixes and anime show fixes (#466)

* Fixed anime identification

* Improved jackett accuracy

* Typo

* Rewrote some of trakt indexer added new class method to show that reindex values accross episodes and seasons. Fixed weird titles

* fixed propagate

* Removed debug

* Removed year param, now determine manually as it gives more accurate results

* Changed to show year instead of air year for each item

* Added back in validation

* Created a new function to grab the season year. Changed jackett matching to support it as well.

* Typo

* Fixed poetry lock hash

---------

Co-authored-by: Joshua <joshazwanink14@gmail.com>
Co-authored-by: Spoked <5782630+dreulavelle@users.noreply.github.com>

* fix: sort imports

* fix: revert jackett. plex watchlist rss feeds broken.

---------

Co-authored-by: Spoked <Spoked@localhost>
Co-authored-by: iPromKnight <156901906+iPromKnight@users.noreply.github.com>
Co-authored-by: dextrous0z <139093885+dextrous0z@users.noreply.github.com>
Co-authored-by: Joshua <joshazwanink14@gmail.com>

VERSION

@@ -1 +1 @@
-0.7.4
+0.7.5

backend/controllers/default.py

@@ -1,6 +1,10 @@
+import time
+
 import requests
 from fastapi import APIRouter, HTTPException, Request
 from program.content.trakt import TraktContent
+from program.media.state import States
+from program.scrapers import Scraping
 from program.settings.manager import settings_manager
 
 router = APIRouter(
@@ -90,3 +94,60 @@ async def trakt_oauth_callback(code: str, request: Request):
         return {"success": True, "message": "OAuth token obtained successfully"}
     else:
         raise HTTPException(status_code=400, detail="Failed to obtain OAuth token")
+
+
+@router.get("/stats")
+async def get_stats(request: Request):
+    payload = {}
+
+    total_items = len(request.app.program.media_items._items)
+    total_movies = len(request.app.program.media_items._movies)
+    total_shows = len(request.app.program.media_items._shows)
+    total_seasons = len(request.app.program.media_items._seasons)
+    total_episodes = len(request.app.program.media_items._episodes)
+
+    _incomplete_items = request.app.program.media_items.get_incomplete_items()
+
+    incomplete_retries = {}
+    for _, item in _incomplete_items.items():
+        incomplete_retries[item.log_string] = item.scraped_times
+
+    states = {}
+    for state in States:
+        states[state] = request.app.program.media_items.count(state)
+
+    payload["total_items"] = total_items
+    payload["total_movies"] = total_movies
+    payload["total_shows"] = total_shows
+    payload["total_seasons"] = total_seasons
+    payload["total_episodes"] = total_episodes
+    payload["incomplete_items"] = len(_incomplete_items)
+    payload["incomplete_retries"] = incomplete_retries
+    payload["states"] = states
+
+    return {"success": True, "data": payload}
+
+@router.get("/scrape/{item_id:path}")
+async def scrape_item(item_id: str, request: Request):
+    item = request.app.program.media_items.get_item(item_id)
+    if item is None:
+        raise HTTPException(status_code=404, detail="Item not found")
+
+    scraper = request.app.program.services.get(Scraping)
+    if scraper is None:
+        raise HTTPException(status_code=404, detail="Scraping service not found")
+
+    time_now = time.time()
+    scraped_results = scraper.scrape(item, log=False)
+    time_end = time.time()
+    duration = time_end - time_now
+
+    results = {}
+    for hash, torrent in scraped_results.items():
+        results[hash] = {
+            "title": torrent.data.parsed_title,
+            "raw_title": torrent.raw_title,
+            "rank": torrent.rank,
+        }
+
+    return {"success": True, "total": len(results), "duration": round(duration, 3), "results": results}

backend/main.py

@@ -9,6 +9,7 @@
 import uvicorn
 from controllers.default import router as default_router
 from controllers.items import router as items_router
+
 # from controllers.metrics import router as metrics_router
 from controllers.settings import router as settings_router
 from controllers.tmdb import router as tmdb_router

backend/program/content/plex_watchlist.py

@@ -16,17 +16,20 @@ def __init__(self):
         self.key = "plex_watchlist"
         self.rss_enabled = False
         self.settings = settings_manager.settings.content.plex_watchlist
+        self.token = settings_manager.settings.updaters.plex.token
         self.initialized = self.validate()
         if not self.initialized:
             return
-        self.token = settings_manager.settings.plex.token
         self.recurring_items = set()
         logger.success("Plex Watchlist initialized!")
 
     def validate(self):
         if not self.settings.enabled:
             logger.warning("Plex Watchlists is set to disabled.")
             return False
+        if not self.token:
+            logger.error("Plex token is not set!")
+            return False
         if self.settings.rss:
             for rss_url in self.settings.rss:
                 try:
@@ -68,6 +71,7 @@ def run(self) -> Generator[Union[Movie, Show, Season, Episode], None, None]:
             self.recurring_items.add(imdb_id)
 
         yield items
+
     def _get_items_from_rss(self) -> Generator[MediaItem, None, None]:
         """Fetch media from Plex RSS Feeds."""
         for rss_url in self.settings.rss:
@@ -76,8 +80,7 @@ def _get_items_from_rss(self) -> Generator[MediaItem, None, None]:
                 if not response.is_ok:
                     logger.error(f"Failed to fetch Plex RSS feed from {rss_url}: HTTP {response.status_code}")
                     continue
-                for item in response.data.items:
-                    yield from self._extract_imdb_ids(item.guids)
+                yield self._extract_imdb_ids(response.data.channel.item.guid)
             except Exception as e:
                 logger.error(f"An unexpected error occurred while fetching Plex RSS feed from {rss_url}: {e}")
 
@@ -102,7 +105,7 @@ def _get_items_from_watchlist(self) -> Generator[MediaItem, None, None]:
     @staticmethod
     def _ratingkey_to_imdbid(ratingKey: str) -> str:
         """Convert Plex rating key to IMDb ID"""
-        token = settings_manager.settings.plex.token
+        token = settings_manager.settings.updaters.plex.token
         filter_params = "includeGuids=1&includeFields=guid,title,year&includeElements=Guid"
         url = f"https://metadata.provider.plex.tv/library/metadata/{ratingKey}?X-Plex-Token={token}&{filter_params}"
         response = get(url)

backend/program/indexers/trakt.py

@@ -1,7 +1,7 @@
 """Trakt updater module"""
 
 from datetime import datetime, timedelta
-from typing import Generator, List, Optional, Union
+from typing import Generator, Optional, Union
 
 from program.media.item import Episode, MediaItem, Movie, Season, Show
 from program.settings.manager import settings_manager
@@ -17,39 +17,47 @@ class TraktIndexer:
 
     def __init__(self):
         self.key = "traktindexer"
-        self.ids = []
         self.initialized = True
         self.settings = settings_manager.settings.indexer
 
-    def copy_items(self, itema: MediaItem, itemb: MediaItem):
+    def copy_items(self, itema: MediaItem, itemb: MediaItem) -> MediaItem:
         if isinstance(itema, Show) and isinstance(itemb, Show):
-            for (seasona, seasonb) in zip(itema.seasons, itemb.seasons):
-                for (episodea, episodeb) in zip(seasona.episodes, seasonb.episodes):
-                    episodeb.set("update_folder", episodea.update_folder)
-                    episodeb.set("symlinked", episodea.symlinked)
-                    episodeb.set("is_anime", episodea.is_anime)
+            for seasona, seasonb in zip(itema.seasons, itemb.seasons):
+                for episodea, episodeb in zip(seasona.episodes, seasonb.episodes):
+                    self._copy_episode_attributes(episodea, episodeb)
         elif isinstance(itema, Movie) and isinstance(itemb, Movie):
-            itemb.set("update_folder", itema.update_folder)
-            itemb.set("symlinked", itema.symlinked)
-            itemb.set("is_anime", itema.is_anime)
+            self._copy_movie_attributes(itema, itemb)
         return itemb
-
+
+    @staticmethod
+    def _copy_episode_attributes(source: Episode, target: Episode) -> None:
+        target.update_folder = source.update_folder
+        target.symlinked = source.symlinked
+        target.is_anime = source.is_anime
+
+    @staticmethod
+    def _copy_movie_attributes(source: Movie, target: Movie) -> None:
+        target.update_folder = source.update_folder
+        target.symlinked = source.symlinked
+        target.is_anime = source.is_anime
+
     def run(self, in_item: MediaItem) -> Generator[Union[Movie, Show, Season, Episode], None, None]:
         """Run the Trakt indexer for the given item."""
         if not in_item:
             logger.error("Item is None")
             return
-        if (imdb_id := in_item.imdb_id) is None:
-            logger.error(f"Item {item.log_string} does not have an imdb_id, cannot index it")
+        if not (imdb_id := in_item.imdb_id):
+            logger.error(f"Item {in_item.log_string} does not have an imdb_id, cannot index it")
             return
-
-        item = create_item_from_imdb_id(imdb_id)
 
+        item = create_item_from_imdb_id(imdb_id)
         if not isinstance(item, MediaItem):
             logger.error(f"Failed to get item from imdb_id: {imdb_id}")
             return
+
         if isinstance(item, Show):
             self._add_seasons_to_show(item, imdb_id)
+
         item = self.copy_items(in_item, item)
         item.indexed_at = datetime.now()
         yield item
@@ -65,7 +73,7 @@ def should_submit(item: MediaItem) -> bool:
             interval = timedelta(seconds=settings.update_interval)
             return datetime.now() - item.indexed_at > interval
         except Exception:
-            logger.error(f"Failed to parse date: {item.indexed_at} with format: {interval}")
+            logger.error(f"Failed to parse date: {item.indexed_at}")
             return False
 
     @staticmethod
@@ -78,46 +86,49 @@ def _add_seasons_to_show(show: Show, imdb_id: str):
         if not imdb_id or not imdb_id.startswith("tt"):
             logger.error(f"Item {show.log_string} does not have an imdb_id, cannot index it")
             return
-
+
+
         seasons = get_show(imdb_id)
         for season in seasons:
             if season.number == 0:
                 continue
-            season_item = _map_item_from_data(season, "season", show.genres)
+            season_item = _map_item_from_data(season, "season")
             if season_item:
-                for episode in season.episodes:
-                    episode_item = _map_item_from_data(episode, "episode", show.genres)
+                for episode_data in season.episodes:
+                    episode_item = _map_item_from_data(episode_data, "episode")
                     if episode_item:
                         season_item.add_episode(episode_item)
                 show.add_season(season_item)
 
+        # Propagate important global attributes to seasons and episodes
+        show.propagate_attributes_to_childs()
 
-def _map_item_from_data(data, item_type: str, show_genres: List[str] = None) -> Optional[MediaItem]:
+def _map_item_from_data(data, item_type: str) -> Optional[MediaItem]:
     """Map trakt.tv API data to MediaItemContainer."""
     if item_type not in ["movie", "show", "season", "episode"]:
-        logger.debug(f"Unknown item type {item_type} for {data.title} not found in list of acceptable items")
+        logger.debug(f"Unknown item type {item_type} for {data.title}")
         return None
 
     formatted_aired_at = _get_formatted_date(data, item_type)
-    genres = getattr(data, "genres", None) or show_genres
+    year = getattr(data, "year", None) or (formatted_aired_at.year if formatted_aired_at else None)
 
     item = {
         "title": getattr(data, "title", None),
-        "year": getattr(data, "year", None),
+        "year": year,
         "status": getattr(data, "status", None),
         "aired_at": formatted_aired_at,
         "imdb_id": getattr(data.ids, "imdb", None),
         "tvdb_id": getattr(data.ids, "tvdb", None),
         "tmdb_id": getattr(data.ids, "tmdb", None),
-        "genres": genres,
+        "genres": getattr(data, "genres", None),
         "network": getattr(data, "network", None),
         "country": getattr(data, "country", None),
         "language": getattr(data, "language", None),
-        "requested_at": datetime.now(),
+        "requested_at": datetime.now(),    
     }
-
+        
     item["is_anime"] = (
-        ("anime" in genres or "animation" in genres) if genres
+        ("anime" in item['genres'] or "animation" in item['genres']) if item['genres']
         and item["country"] in ("jp", "kr")
         else False
     )
@@ -134,17 +145,14 @@ def _map_item_from_data(data, item_type: str, show_genres: List[str] = None) ->
             item["number"] = data.number
             return Episode(item)
         case _:
-            logger.error(f"Unknown item type {item_type} for {data.title} not found in list of acceptable items")
+            logger.error(f"Failed to create item from data: {data}")
             return None
 
-
 def _get_formatted_date(data, item_type: str) -> Optional[datetime]:
     """Get the formatted aired date from the data."""
-    if item_type in ["show", "season", "episode"] and (first_aired := getattr(data, "first_aired", None)):
-        return datetime.strptime(first_aired, "%Y-%m-%dT%H:%M:%S.%fZ")
-    if item_type == "movie" and (released := getattr(data, "released", None)):
-        return datetime.strptime(released, "%Y-%m-%d")
-    return None
+    date_str = getattr(data, "first_aired" if item_type in ["show", "season", "episode"] else "released", None)
+    date_format = "%Y-%m-%dT%H:%M:%S.%fZ" if item_type in ["show", "season", "episode"] else "%Y-%m-%d"
+    return datetime.strptime(date_str, date_format) if date_str else None
 
 
 def get_show(imdb_id: str) -> dict:
@@ -159,37 +167,16 @@ def create_item_from_imdb_id(imdb_id: str) -> Optional[MediaItem]:
     url = f"https://api.trakt.tv/search/imdb/{imdb_id}?extended=full"
     response = get(url, additional_headers={"trakt-api-version": "2", "trakt-api-key": CLIENT_ID})
     if not response.is_ok or not response.data:
-        logger.error(f"Failed to create item using imdb id: {imdb_id}")  # This returns an empty list for response.data
-        return None
-
-    def find_first(preferred_types, data):
-        for type in preferred_types:
-            for d in data:
-                if d.type == type:
-                    return d
+        logger.error(f"Failed to create item using imdb id: {imdb_id}")
         return None
 
-    data = find_first(["show", "movie", "season", "episode"], response.data)
-    if data:
-        return _map_item_from_data(getattr(data, data.type), data.type)
-    return None
+    data = next((d for d in response.data if d.type in ["show", "movie", "season", "episode"]), None)
+    return _map_item_from_data(getattr(data, data.type), data.type) if data else None
 
 def get_imdbid_from_tmdb(tmdb_id: str) -> Optional[str]:
     """Wrapper for trakt.tv API search method."""
     url = f"https://api.trakt.tv/search/tmdb/{tmdb_id}?extended=full"
     response = get(url, additional_headers={"trakt-api-version": "2", "trakt-api-key": CLIENT_ID})
     if not response.is_ok or not response.data:
         return None
-    imdb_id = get_imdb_id_from_list(response.data)
-    if imdb_id:
-        return imdb_id
-    logger.error(f"Failed to fetch imdb_id for tmdb_id: {tmdb_id}")
-    return None
-
-def get_imdb_id_from_list(namespaces):
-    for ns in namespaces:
-        if ns.type == 'movie':
-            return ns.movie.ids.imdb
-        elif ns.type == 'show':
-            return ns.show.ids.imdb
-    return None
+    return next((ns.movie.ids.imdb if ns.type == 'movie' else ns.show.ids.imdb for ns in response.data if ns.type in ['movie', 'show']), None)