getting a bit closer to success (import using required topics)

domain/Migrator.py

@@ -1,7 +1,6 @@
 
 
-# import giteapy
-import giteapy.giteapy
+import giteapy
 import logging
 import sys
 
@@ -117,9 +116,8 @@ class Migrator:
 		api_source: giteapy.OrganizationApi
 		api_destination: giteapy.OrganizationApi
 		
-		api_source_repos = self._get_repo_api(hostname=self.__source_host, port=self.__source_port, token=self.__source_token)
-		
-		source_repos = api_source.org_list_repos(source_org, page=0, limit=1000000)
+		# Grab all org repos
+		source_repos = self._fetch_all_org_repos(org=source_org)
 		self.__logger.info(f"Found {len(source_repos)} repos on source:")
 		for repo in source_repos:
 			repo: giteapy.Repository
@@ -127,6 +125,9 @@ class Migrator:
 		
 		print()
 		
+		# Filter
+		source_repos = self._filter_repos_for_required_topics(repos=source_repos, topics=source_topics)
+		
 		repos_migrate = []
 		repos_ignore = []
 		go_right_now = False
@@ -134,20 +135,6 @@ class Migrator:
 			
 			repo: giteapy.Repository
 			
-			repo_topics = api_source_repos.repo_list_topics(owner=repo.owner.login, repo=repo.name)
-			repo_topics = repo_topics.topics
-			self.__logger.error(f"Repo topics: {repo_topics}")
-			
-			if self._check_source_repo_topics(repo=repo, topics=source_topics) is False:
-				repos_ignore.append(repo)
-				self.__logger.info(
-					f"Ignoring repo because it doesn't have all required tags."
-					f"\n> Repo: {repo.full_name}"
-					f"\n> Has: {repo_topics}"
-					f"\n> Needs: {source_topics}"
-				)
-				continue
-			
 			while True:
 				
 				if interactive:
@@ -219,9 +206,74 @@ class Migrator:
 			else:
 				self.__logger.info("Confirmation not received; Won't do anything.")
 	
-	def _check_source_repo_topics(self, repo: giteapy.Repository, topics: list[str]) -> bool:
+	def _fetch_all_org_repos(self, org: str):
 		
-		return False
+		api_source, api_destination = self._get_org_apis()
+		api_source: giteapy.OrganizationApi
+		
+		source_repos = []
+		
+		page = 0
+		while True:
+			
+			page += 1  # Starts at 1 for some reason
+			source_repos_page = api_source.org_list_repos(org, page=page, limit=25)
+			
+			if len(source_repos_page) == 0:
+				break
+			
+			source_repos.extend(source_repos_page)
+		
+		return source_repos
+	
+	def _filter_repos_for_required_topics(self, repos: list[giteapy.Repository], topics: list[str]) -> list[giteapy.Repository]:
+		
+		self.__logger.info(f"Filtering source repos for required topics: {topics}")
+		
+		repos_keep = []
+		repos_reject = []
+		repo_topics = {}
+		
+		api_source_repos = self._get_repo_api(
+			hostname=self.__source_host, port=self.__source_port,
+			token=self.__source_token
+		)
+		
+		for repo in repos:
+			
+			repo_topics[repo.id] = api_source_repos.repo_list_topics(owner=repo.owner.login, repo=repo.name)
+			repo_topics[repo.id] = repo_topics[repo.id].topics
+			
+			self.__logger.error(f"Repo topics: {repo_topics[repo.id]}")
+			
+			if self._check_required_topics(topics_present=repo_topics[repo.id], topics_required=topics):
+				repos_keep.append(repo)
+			else:
+				repos_reject.append(repo)
+		
+		self.__logger.info(f"Keeping the following repos because they contain all required topics ({topics}):")
+		if len(repos_keep) > 0:
+			for repo in repos_keep:
+				self.__logger.info(f"> {repo.full_name}")
+		else:
+			self.__logger.info("> None")
+		
+		self.__logger.info("Rejecting the following repos because they don't contain all required topics:")
+		if len(repos_reject) > 0:
+			for repo in repos_reject:
+				self.__logger.info(f"> {repo.full_name} ({repo_topics[repo.id]})")
+		else:
+			self.__logger.info("> None")
+		
+		return repos_keep
+	
+	def _check_required_topics(self, topics_present: list[str], topics_required: list[str]) -> bool:
+		
+		for topic in topics_required:
+			if topic not in topics_present:
+				return False
+		
+		return True
 	
 	def _migrate_repos(
 			self,