getting a bit closer to success (import using required topics)

Pipfile

@@ -4,6 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
+giteapy-soteria = {git = "https://github.com/Yousif-CS/giteapy.git"}
 
 [dev-packages]
 

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "fedbd2ab7afd84cf16f128af0619749267b62277b4cb6989ef16d4bef6e4eef2"
+            "sha256": "1dc9e96fd5a12468ed7d0869b11b9fbca2464e4b806fb8b9c17391a41b6f0eb8"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -15,6 +15,43 @@
             }
         ]
     },
-    "default": {},
+    "default": {
+        "certifi": {
+            "hashes": [
+                "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3",
+                "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==2022.12.7"
+        },
+        "giteapy-soteria": {
+            "git": "https://github.com/Yousif-CS/giteapy.git",
+            "ref": "e0a089bdfb7ef6130b43727c50e78f176379db20"
+        },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
+                "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "version": "==2.8.2"
+        },
+        "six": {
+            "hashes": [
+                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "version": "==1.16.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72",
+                "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+            "version": "==1.26.14"
+        }
+    },
     "develop": {}
 }

domain/Migrator.py

@@ -1,7 +1,6 @@
 
 
-# import giteapy
+import giteapy
-import giteapy.giteapy
 import logging
 import sys
 
@@ -117,9 +116,8 @@ class Migrator:
 		api_source: giteapy.OrganizationApi
 		api_destination: giteapy.OrganizationApi
 		
-		api_source_repos = self._get_repo_api(hostname=self.__source_host, port=self.__source_port, token=self.__source_token)
+		# Grab all org repos
-		
+		source_repos = self._fetch_all_org_repos(org=source_org)
-		source_repos = api_source.org_list_repos(source_org, page=0, limit=1000000)
 		self.__logger.info(f"Found {len(source_repos)} repos on source:")
 		for repo in source_repos:
 			repo: giteapy.Repository
@@ -127,6 +125,9 @@ class Migrator:
 		
 		print()
 		
+		# Filter
+		source_repos = self._filter_repos_for_required_topics(repos=source_repos, topics=source_topics)
+		
 		repos_migrate = []
 		repos_ignore = []
 		go_right_now = False
@@ -134,20 +135,6 @@ class Migrator:
 			
 			repo: giteapy.Repository
 			
-			repo_topics = api_source_repos.repo_list_topics(owner=repo.owner.login, repo=repo.name)
-			repo_topics = repo_topics.topics
-			self.__logger.error(f"Repo topics: {repo_topics}")
-			
-			if self._check_source_repo_topics(repo=repo, topics=source_topics) is False:
-				repos_ignore.append(repo)
-				self.__logger.info(
-					f"Ignoring repo because it doesn't have all required tags."
-					f"\n> Repo: {repo.full_name}"
-					f"\n> Has: {repo_topics}"
-					f"\n> Needs: {source_topics}"
-				)
-				continue
-			
 			while True:
 				
 				if interactive:
@@ -219,10 +206,75 @@ class Migrator:
 			else:
 				self.__logger.info("Confirmation not received; Won't do anything.")
 	
-	def _check_source_repo_topics(self, repo: giteapy.Repository, topics: list[str]) -> bool:
+	def _fetch_all_org_repos(self, org: str):
 		
+		api_source, api_destination = self._get_org_apis()
+		api_source: giteapy.OrganizationApi
+		
+		source_repos = []
+		
+		page = 0
+		while True:
+			
+			page += 1  # Starts at 1 for some reason
+			source_repos_page = api_source.org_list_repos(org, page=page, limit=25)
+			
+			if len(source_repos_page) == 0:
+				break
+			
+			source_repos.extend(source_repos_page)
+		
+		return source_repos
+	
+	def _filter_repos_for_required_topics(self, repos: list[giteapy.Repository], topics: list[str]) -> list[giteapy.Repository]:
+		
+		self.__logger.info(f"Filtering source repos for required topics: {topics}")
+		
+		repos_keep = []
+		repos_reject = []
+		repo_topics = {}
+		
+		api_source_repos = self._get_repo_api(
+			hostname=self.__source_host, port=self.__source_port,
+			token=self.__source_token
+		)
+		
+		for repo in repos:
+			
+			repo_topics[repo.id] = api_source_repos.repo_list_topics(owner=repo.owner.login, repo=repo.name)
+			repo_topics[repo.id] = repo_topics[repo.id].topics
+			
+			self.__logger.error(f"Repo topics: {repo_topics[repo.id]}")
+			
+			if self._check_required_topics(topics_present=repo_topics[repo.id], topics_required=topics):
+				repos_keep.append(repo)
+			else:
+				repos_reject.append(repo)
+		
+		self.__logger.info(f"Keeping the following repos because they contain all required topics ({topics}):")
+		if len(repos_keep) > 0:
+			for repo in repos_keep:
+				self.__logger.info(f"> {repo.full_name}")
+		else:
+			self.__logger.info("> None")
+		
+		self.__logger.info("Rejecting the following repos because they don't contain all required topics:")
+		if len(repos_reject) > 0:
+			for repo in repos_reject:
+				self.__logger.info(f"> {repo.full_name} ({repo_topics[repo.id]})")
+		else:
+			self.__logger.info("> None")
+		
+		return repos_keep
+	
+	def _check_required_topics(self, topics_present: list[str], topics_required: list[str]) -> bool:
+		
+		for topic in topics_required:
+			if topic not in topics_present:
 				return False
 		
+		return True
+	
 	def _migrate_repos(
 			self,
 			destination_org_name: str,