Track+log some migration failures instead of actually failing

.python-version

@@ -0,0 +1 @@
+3.12.1

Pipfile

@@ -4,9 +4,10 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-giteapy = "*"
+giteapy-soteria = {git = "https://github.com/Yousif-CS/giteapy.git"}
 
 [dev-packages]
 
 [requires]
-python_version = "3.10"
+python_version = "3.12"
+

Pipfile.lock

@@ -1,11 +1,11 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "824c02127733d501ae21f3e5ccc7ee72fc4b2fe92bbca23c77ebd204d294b0fc"
+            "sha256": "5041607b8b692ebdc03484547b2d4336083196ff75b38c3140b608a7d59abaf8"
         },
         "pipfile-spec": 6,
         "requires": {
-            "python_version": "3.10"
+            "python_version": "3.12"
         },
         "sources": [
             {
@@ -18,25 +18,22 @@
     "default": {
         "certifi": {
             "hashes": [
-                "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3",
-                "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"
+                "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1",
+                "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2022.12.7"
+            "version": "==2023.11.17"
         },
-        "giteapy": {
-            "hashes": [
-                "sha256:2078c802a4626bf311e911c969c34b7d19fbe9175e2910e1965b24ff69221470"
-            ],
-            "index": "pypi",
-            "version": "==1.0.8"
+        "giteapy-soteria": {
+            "git": "https://github.com/Yousif-CS/giteapy.git",
+            "ref": "e0a089bdfb7ef6130b43727c50e78f176379db20"
         },
         "python-dateutil": {
             "hashes": [
                 "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
                 "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==2.8.2"
         },
         "six": {
@@ -44,16 +41,16 @@
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==1.16.0"
         },
         "urllib3": {
             "hashes": [
-                "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72",
-                "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"
+                "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3",
+                "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
-            "version": "==1.26.14"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.1.0"
         }
     },
     "develop": {}

README.md

@@ -1,2 +1,57 @@
-# gitea-repo-migrator
+# Mike's Gitea Repo Migrator
+
+Just a script to help make it a little easier to migrate an entire organization (with bulk selection) from one Gitea instance to another.
+
+Supports changing destination names a bit, and adding topics to each transferred repo, as well as optional bulk-selected deletion from the source.
+
+by Mike Peralta
+
+Current license: You are free to clone and use this program but all other rights reserved, provided you accept 100% of all liability of any outcome of use/download/etc. this program. Better license coming soon.
+
+## Requirements
+
+* python 3.12
+* pipenv
+
+## Installation
+
+1. Clone this repo and `cd` to its root directory
+
+2. Execute `pipenv install` (and optionally check with `pipenv check`)
+
+## Execution
+
+Ask `pipenv` to execute *main.py* on your behalf with the following:
+
+```console
+
+$ cd /path/to/gitea-repo-migrator
+$ pipenv run python ./main.py --help
+```
+
+Optionally ask for a shell to run multiple times a bit faster:
+
+```console
+
+$ cd /path/to/gitea-repo-migrator
+$ pipenv shell
+$ python ./main.py --help
+$ python ./main.py --help
+```
+
+### App Tokens
+
+You'll need to generate an *Application Token* for both the source and destination servers, and pass the token along to the command line.
+
+### SSL Verification
+
+Pass the long switch `--no-verify-ssl` if any server sits behind a self-signed or wonky SSL certificate.
+
+### Destination Repo Names
+
+You can tweak the destination repo names a bit by using a string that includes `%N%` somewhere. The string `%N%` will expand to the original name. For example, if we use a repo originally named `my-test-repo` with the string `what-%N%`, the resulting destination repo name will be `what-my-test-repo`. This string will be recomputed for every migrated repo.
+
+### Destination Topics
+
+Topics will be duplicated from all source repos to their corresponding destination repos. You can specify additional topics with the `--destination-topic` switch. For example, to add the topic `migrated` to every repo, pass the switch `--destination-topic migrated`.
 

domain/Migrator.py

@@ -4,6 +4,8 @@ import giteapy
 import logging
 import sys
 
+import certifi
+
 
 class Migrator:
 	
@@ -101,15 +103,37 @@ class Migrator:
 		
 		self.__verify_ssl = b
 	
+	def set_ca_bundle(self, bundle_path: str):
+		
+		self.__logger.info("Setting certificate bundle path")
+		
+		# Hacky but oh well
+		self.__logger.info(f"Old path: {certifi.where()}")
+		certifi.core._CACERT_PATH = bundle_path
+		self.__logger.info(f"New path: {certifi.where()}")
+	
 	def migrate_entire_org(
 			self,
-		source_org: str,
-		destination_org: str, destination_repo_name: str, destination_topics: list
+			interactive: bool = True,
+			source_org: str = None, source_topics: list[str] = None,
+			destination_org: str = None, destination_repo_name: str = None, destination_topics: list = None,
+			do_destination_copy_topics: bool = True
 	):
+		assert source_org is not None, "Source org must be specified"
+		assert destination_org is not None, "Destination org must be specified"
+		assert destination_repo_name is not None, "Destination repo name must be specified"
+		assert destination_topics is not None, "Destination topics must be specified"
+		assert do_destination_copy_topics is not None, "Destination directive to copy source topics should be specified"
 		
-		api_source, api_destination = self._get_org_apis()
+		# api_source, api_destination = self._get_org_apis()
+		api_source: giteapy.OrganizationApi
+		api_destination: giteapy.OrganizationApi
 		
-		source_repos = api_source.org_list_repos(source_org)
+		# Tattle on certify
+		self.__logger.info(f"Certifi is currently using CA bundle: {certifi.where()}")
+		
+		# Grab all org repos
+		source_repos = self._fetch_all_org_repos(org=source_org)
 		self.__logger.info(f"Found {len(source_repos)} repos on source:")
 		for repo in source_repos:
 			repo: giteapy.Repository
@@ -117,6 +141,10 @@ class Migrator:
 		
 		print()
 		
+		# Filter
+		source_repos = self._filter_repos_for_required_topics(repos=source_repos, topics=source_topics)
+		print()
+		
 		repos_migrate = []
 		repos_ignore = []
 		go_right_now = False
@@ -126,8 +154,11 @@ class Migrator:
 			
 			while True:
 				
+				if interactive:
 					response = input(f"Migrate repo #{repo.id} \"{repo.full_name}\" ? (Y)es, (N)o, (G)o right now, (Q)uit ==> ")
 					response = response.lower()
+				else:
+					response = "y"
 				
 				valid_input = True
 				if response == "y":
@@ -159,7 +190,9 @@ class Migrator:
 			for repo in repos_migrate:
 				repo: giteapy.Repository
 				destination_name = self._make_destination_repo_name(pattern=destination_repo_name, repo=repo)
-				self.__logger.info(f"#{repo.id} \"{repo.name}\" ==> \"{destination_name}\"")
+				self.__logger.info(
+					f"#{repo.id} \"{repo.name}\"\n> \"{destination_name}\""
+				)
 		else:
 			self.__logger.info("No repos marked to migrate")
 		
@@ -179,24 +212,115 @@ class Migrator:
 			if confirmation == "MIGRATE":
 				
 				self.__logger.info("Confirmation received; Processing ... ")
-				source_repos_successful = self._migrate_repos(
+				source_repos_successful, source_repos_failed = self._migrate_repos(
 					destination_org_name=destination_org,
 					destination_repo_name=destination_repo_name,
 					destination_topics=destination_topics,
+					do_destination_copy_topics=do_destination_copy_topics,
 					repos=repos_migrate
 				)
-				self.__logger.info(f"{len(source_repos_successful)} of {len(repos_migrate)} repos successfully migrated.")
+				self.__logger.info(
+					f"{len(source_repos_successful)} of {len(repos_migrate)} repos successfully migrated."
+				)
+				if len(source_repos_failed) > 0:
+					self.__logger.error(f"Failed to migrate {len(source_repos_failed)} repos:")
+					for repo, exception in source_repos_failed:
+						self.__logger.error(
+							f"> {repo.name}"
+						)
+					self.__logger.error(f"Captured exception data:")
+					for repo, exception in source_repos_failed:
+						self.__logger.error(
+							f"Failed to migrate repo: {repo.name}\n> {exception}"
+						)
 				
 				self._delete_migrated_repos(source_org_name=source_org, repos=source_repos_successful)
 				
 			else:
 				self.__logger.info("Confirmation not received; Won't do anything.")
 	
+	def _fetch_all_org_repos(self, org: str):
+		
+		api_source, api_destination = self._get_org_apis()
+		api_source: giteapy.OrganizationApi
+		
+		source_repos = []
+		
+		page = 0
+		while True:
+			
+			page += 1  # Starts at 1 for some reason
+			source_repos_page = api_source.org_list_repos(org, page=page, limit=25)
+			
+			if len(source_repos_page) == 0:
+				break
+			
+			source_repos.extend(source_repos_page)
+		
+		return source_repos
+	
+	def _filter_repos_for_required_topics(
+			
+			self,
+			repos: list[giteapy.Repository],
+			topics: list[str]
+			
+	) -> list[giteapy.Repository]:
+		
+		self.__logger.info(f"Filtering source repos for required topics: {topics}")
+		
+		repos_keep = []
+		repos_reject = []
+		repo_topics = {}
+		
+		api_source_repos = self._get_repo_api(
+			hostname=self.__source_host, port=self.__source_port,
+			token=self.__source_token
+		)
+		
+		for repo in repos:
+			
+			repo_topics[repo.id] = api_source_repos.repo_list_topics(owner=repo.owner.login, repo=repo.name)
+			repo_topics[repo.id] = repo_topics[repo.id].topics
+			
+			if self._check_required_topics(topics_present=repo_topics[repo.id], topics_required=topics):
+				repos_keep.append(repo)
+			else:
+				repos_reject.append(repo)
+		
+		self.__logger.info("")
+		self.__logger.info(f"\nKeeping {len(repos_keep)} repos because they contain all required topics ({topics}):")
+		if len(repos_keep) > 0:
+			for repo in repos_keep:
+				self.__logger.info(f"> {repo.full_name}")
+		else:
+			self.__logger.info("> None")
+		
+		self.__logger.info("")
+		self.__logger.info(f"Rejecting {len(repos_reject)} repos because they don't contain all required topics:")
+		if len(repos_reject) > 0:
+			for repo in repos_reject:
+				self.__logger.info(f"> {repo.full_name} ({repo_topics[repo.id]})")
+		else:
+			self.__logger.info("> None")
+		
+		return repos_keep
+	
+	@staticmethod
+	def _check_required_topics(topics_present: list[str], topics_required: list[str]) -> bool:
+		
+		for topic in topics_required:
+			if topic not in topics_present:
+				return False
+		
+		return True
+	
 	def _migrate_repos(
 			self,
 			destination_org_name: str,
 			destination_repo_name: str,
 			destination_topics: list,
+			do_destination_copy_topics: bool,
 			repos: list
 	):
 		
@@ -204,16 +328,26 @@ class Migrator:
 		
 		destination_org = api_destination.org_get(org=destination_org_name)
 		destination_org: giteapy.Organization
-		
 		self.__logger.info(f"Destination organization: {destination_org.full_name}")
 		
+		api_source_repos = self._get_repo_api(
+			hostname=self.__source_host, port=self.__source_port,
+			token=self.__source_token
+		)
+		
 		source_repos_successful = []
+		source_repos_failed = []
 		for source_repo in repos:
 			
 			source_repo: giteapy.Repository
 			
 			this_destination_repo_name = destination_repo_name.replace("%N%", source_repo.name)
 			
+			self.__logger.info(f"Migrating: {source_repo.name} ==> {this_destination_repo_name}")
+			
+			source_repo_topics = api_source_repos.repo_list_topics(owner=source_repo.owner.login, repo=source_repo.name)
+			source_repo_topics = source_repo_topics.topics
+			
 			migrate_body = giteapy.MigrateRepoForm(
 				mirror=False,
 				clone_addr=source_repo.clone_url,
@@ -231,18 +365,45 @@ class Migrator:
 			self.__logger.debug("Migrate body:")
 			self.__logger.debug(migrate_body)
 			
+			try:
 				destination_api = self._get_repo_api(
 					hostname=self.__destination_host,
 					port=self.__destination_port,
 					token=self.__destination_token,
 				)
+			except giteapy.rest.ApiException as e:
+				self.__logger.error(f"Failed to generate destination API: {e}")
+				source_repos_failed.append(
+					(source_repo, e)
+				)
+				continue
+			
+			try:
 				repo_new = destination_api.repo_migrate(body=migrate_body)
+			except giteapy.rest.ApiException as e:
+				self.__logger.error(f"Failed to execute repo migration request via API: {e}")
+				source_repos_failed.append(
+					(source_repo, e)
+				)
+				continue
+			
 			self.__logger.debug(f"Migration result: {repo_new}")
 			repo_new: giteapy.Repository
 			
 			assert repo_new.name == this_destination_repo_name,\
 				"New repository didn't end up with the correct name. Failure?"
 			
+			# Copy source topics?
+			if do_destination_copy_topics:
+				for topic in source_repo_topics:
+					self.__logger.debug(f"Appending source topic to new repo: {topic}")
+					destination_api.repo_add_topc(
+						owner=destination_org.username,
+						repo=repo_new.name,
+						topic=topic,
+					)
+			
+			# Add specified topics
 			for topic in destination_topics:
 				self.__logger.debug(f"Appending topic to new repo: {topic}")
 				destination_api.repo_add_topc(
@@ -253,10 +414,14 @@ class Migrator:
 			
 			source_repos_successful.append(source_repo)
 		
-		return source_repos_successful
+		return source_repos_successful, source_repos_failed
 	
 	def _delete_migrated_repos(self, source_org_name: str, repos: list[giteapy.Repository]):
 		
+		if len(repos) == 0:
+			self.__logger.warning(f"Cannot delete any migrated repos because none were successful!")
+			return
+		
 		repo_api = self._get_repo_api(
 			hostname=self.__source_host,
 			port=self.__source_port,
@@ -265,7 +430,7 @@ class Migrator:
 		repo_api: giteapy.RepositoryApi
 		
 		self.__logger.info("")
-		self.__logger.info("Can now delete the following successfully migrated repos:")
+		self.__logger.info(f"Can now delete {len(repos)} successfully migrated repos:")
 		for r in repos:
 			self.__logger.info(f"> #{r.id} \"{r.full_name}\" ==> {r.clone_url}")
 		

main.py

@@ -23,7 +23,7 @@ def main():
 		dest="source_port",
 		required=False,
 		default=None,
-		help="Port of the source server"
+		help="Port of the source server. Requests will use https (not ssh), so you probably don't want to change this."
 	)
 	parser.add_argument(
 		"--source-token",
@@ -37,6 +37,14 @@ def main():
 		required=True,
 		help="Name of the source organization"
 	)
+	parser.add_argument(
+		"--source-required-topic", "--source-topic",
+		dest="source_topics",
+		default=[],
+		action="append",
+		help="Specify zero or more topics required topics for source repositories."
+		" Any repository that doesn't have all required topics will be skipped"
+	)
 	
 	parser.add_argument(
 		"--destination-hostname", "--dest-hostname", "--destination-host", "--dest-host",
@@ -49,7 +57,7 @@ def main():
 		dest="destination_port",
 		required=False,
 		default=None,
-		help="Port of the destination server"
+		help="Port of the destination server. Requests will use https (not ssh), so you probably don't want to change this."
 	)
 	parser.add_argument(
 		"--destination-token", "--dest-token",
@@ -68,16 +76,46 @@ def main():
 		"--destination-repo-name", "--destination-name", "--dest-repo-name", "--dest-name",
 		dest="destination_repo_name",
 		default="%N%",
-		help="Specify the destination repository name(s). Use wildcard %N% anywhere to denote the original name"
+		help="Specify the destination repository name(s). Use wildcard %%N%% anywhere to denote the original name"
 	)
 	parser.add_argument(
-		"--destination-add-topic", "--destination-topic", "-dest-add-topic", "--dest-topic",
+		"--destination-add-topic", "--destination-topic", "--dest-add-topic", "--dest-topic",
 		dest="destination_topics",
 		default=[],
 		action="append",
 		help="Specify zero or more topics to add to each destination repository"
 	)
 	
+	parser.add_argument(
+		"--destination-copy-topics", "--dest-copy-topics",
+		dest="do_destination_copy_topics",
+		default=True,
+		action="store_true",
+		help="Destination repos should copy topics from their source."
+	)
+	parser.add_argument(
+		"--no-destination-copy-topics", "--no-dest-copy-topics",
+		dest="do_destination_copy_topics",
+		default=True,
+		action="store_false",
+		help="Destination repos should NOT copy topics from their source."
+	)
+	
+	parser.add_argument(
+		"--interactive",
+		dest="interactive",
+		default=True,
+		action="store_true",
+		help="Ask to confirm each migration.",
+	)
+	parser.add_argument(
+		"--no-interactive", "--non-interactive",
+		dest="interactive",
+		default=True,
+		action="store_false",
+		help="Do not ask to confirm each migration; Migrate all repos quickly.",
+	)
+	
 	parser.add_argument(
 		"--no-verify-ssl",
 		dest="verify_ssl",
@@ -86,6 +124,13 @@ def main():
 		help="Don't verify SSL certificates",
 	)
 	
+	parser.add_argument(
+		"--ca-bundle",
+		dest="ca_bundle",
+		default=None,
+		help="Specify the location of your system-wide CA Bundle, in case python is not using it."
+	)
+	
 	args = parser.parse_args()
 	mig = Migrator(
 		source_host=args.source_hostname,
@@ -95,12 +140,19 @@ def main():
 		destination_port=args.destination_port,
 		destination_token=args.destination_token
 	)
+	
 	mig.set_verify_ssl(args.verify_ssl)
+	if args.ca_bundle:
+		mig.set_ca_bundle(args.ca_bundle)
+	
 	mig.migrate_entire_org(
+		interactive=args.interactive,
 		source_org=args.source_org,
+		source_topics=args.source_topics,
 		destination_org=args.destination_org,
 		destination_repo_name=args.destination_repo_name,
-		destination_topics=args.destination_topics
+		destination_topics=args.destination_topics,
+		do_destination_copy_topics=args.do_destination_copy_topics
 	)