Fix CKAN harvester search functionality

- Now return results of all pages, not only the first x rows.
mjanez · Jul 31, 2024 · 3952322 · 3952322
1 parent 3beccce
commit 3952322
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/ckanext/schemingdcat/harvesters/ckan.py b/ckanext/schemingdcat/harvesters/ckan.py
@@ -383,10 +383,11 @@ def _search_for_datasets(self, remote_ckan_base_url, fq_terms=None):
         pkg_dicts = []
         pkg_ids = set()
         previous_content = None
-        url = base_search_url + "?" + urlencode(params)
-        log.debug("Searching for CKAN datasets: %s", url)
 
         while True:
+            url = base_search_url + "?" + urlencode(params)
+            log.debug("Searching for CKAN datasets: %s", url)
+
             try:
                 content = self._get_content(url)
             except ContentFetchError as e:
@@ -429,6 +430,8 @@ def _search_for_datasets(self, remote_ckan_base_url, fq_terms=None):
 
             params["start"] = str(int(params["start"]) + int(params["rows"]))
 
+        log.debug('Number of elements in remote CKAN: %s', len(pkg_dicts))
+
         return pkg_dicts
 
     def fetch_stage(self, harvest_object):