From 17a6422b1b7a3fa74592b448154b01ec91f80579 Mon Sep 17 00:00:00 2001 From: mike Date: Mon, 17 Jun 2024 23:45:40 -0700 Subject: [PATCH] Started refactoring to py-gitea --- .python-version | 2 +- Pipfile | 4 +- Pipfile.lock | 145 ++++++++++++++++++++++++++++++++++++++------- domain/API.py | 36 +++++++++++ domain/Migrator.py | 136 +++++++++++++++++++++++++----------------- 5 files changed, 245 insertions(+), 78 deletions(-) create mode 100644 domain/API.py diff --git a/.python-version b/.python-version index 171a6a9..455808f 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.12.1 +3.12.4 diff --git a/Pipfile b/Pipfile index 8597a9b..453226b 100644 --- a/Pipfile +++ b/Pipfile @@ -4,10 +4,10 @@ verify_ssl = true name = "pypi" [packages] -giteapy-soteria = {git = "https://github.com/Yousif-CS/giteapy.git"} +#py-gitea = "*" +py-gitea = {file = "/home/mike/opt/py-gitea"} [dev-packages] [requires] python_version = "3.12" - diff --git a/Pipfile.lock b/Pipfile.lock index a6ea9bc..cbc993e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5041607b8b692ebdc03484547b2d4336083196ff75b38c3140b608a7d59abaf8" + "sha256": "42107811e88c77bb070866e53f75374682faf2ad34fa0f94450921ecd5ac9e38" }, "pipfile-spec": 6, "requires": { @@ -18,39 +18,142 @@ "default": { "certifi": { "hashes": [ - "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1", - "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474" + "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516", + "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56" ], "markers": "python_version >= '3.6'", - "version": "==2023.11.17" + "version": "==2024.6.2" }, - "giteapy-soteria": { - "git": "https://github.com/Yousif-CS/giteapy.git", - "ref": "e0a089bdfb7ef6130b43727c50e78f176379db20" - }, - "python-dateutil": { + "charset-normalizer": { "hashes": [ - "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", - "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", + "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087", + "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786", + "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8", + "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09", + "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", + "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", + "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e", + "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", + "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898", + "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", + "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3", + "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f", + "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6", + "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", + "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a", + "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73", + "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", + "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714", + "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2", + "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", + "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", + "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d", + "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", + "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6", + "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269", + "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", + "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", + "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a", + "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", + "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77", + "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d", + "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", + "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed", + "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068", + "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac", + "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25", + "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", + "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab", + "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26", + "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", + "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", + "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", + "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5", + "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99", + "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c", + "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d", + "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811", + "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa", + "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a", + "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03", + "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b", + "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04", + "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c", + "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001", + "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", + "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389", + "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99", + "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985", + "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537", + "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238", + "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f", + "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d", + "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", + "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a", + "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143", + "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8", + "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c", + "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5", + "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5", + "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711", + "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", + "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6", + "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", + "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7", + "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4", + "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b", + "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", + "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12", + "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", + "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", + "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8", + "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", + "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b", + "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4", + "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", + "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", + "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33", + "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", + "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", - "version": "==2.8.2" + "markers": "python_full_version >= '3.7.0'", + "version": "==3.3.2" }, - "six": { + "idna": { "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc", + "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", - "version": "==1.16.0" + "markers": "python_version >= '3.5'", + "version": "==3.7" + }, + "immutabledict": { + "hashes": [ + "sha256:d728b2c2410d698d95e6200237feb50a695584d20289ad3379a439aa3d90baba", + "sha256:e003fd81aad2377a5a758bf7e1086cf3b70b63e9a5cc2f46bce8d0a2b4727c5f" + ], + "markers": "python_version >= '3.8' and python_version < '4.0'", + "version": "==4.2.0" + }, + "py-gitea": { + "file": "/home/mike/opt/py-gitea" + }, + "requests": { + "hashes": [ + "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", + "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6" + ], + "markers": "python_version >= '3.8'", + "version": "==2.32.3" }, "urllib3": { "hashes": [ - "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3", - "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54" + "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472", + "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168" ], "markers": "python_version >= '3.8'", - "version": "==2.1.0" + "version": "==2.2.2" } }, "develop": {} diff --git a/domain/API.py b/domain/API.py new file mode 100644 index 0000000..6c471d6 --- /dev/null +++ b/domain/API.py @@ -0,0 +1,36 @@ + + +import gitea + + +class API: + + __DEFAULT_API_PATH = "/api/v1" + + def __init__(self): + + pass + + @staticmethod + def _make_api_base_url(hostname, port): + + base = f"https://{hostname}" + if port is not None: + base += f":{port}" + + return base + + @staticmethod + def factory(hostname, port, token) -> gitea.Gitea: + + url = API._make_api_base_url( + hostname=hostname, + port=port + ) + + g = gitea.Gitea( + gitea_url=url, + token_text=token + ) + + return g diff --git a/domain/Migrator.py b/domain/Migrator.py index a9bae3c..3b7d034 100644 --- a/domain/Migrator.py +++ b/domain/Migrator.py @@ -1,6 +1,9 @@ -import giteapy +from domain.API import API + + +import gitea import logging import sys @@ -9,7 +12,6 @@ import certifi class Migrator: - __DEFAULT_API_PATH = "/api/v1" __REPO_ORIGINAL_NAME_TOKEN = "%N%" def __init__( @@ -31,6 +33,17 @@ class Migrator: self.__destination_host = destination_host self.__destination_port = destination_port self.__destination_token = destination_token + + self.__source_api = API.factory( + hostname=self.__source_host, + port=self.__source_port, + token=self.__source_token + ) + self.__destination_api = API.factory( + hostname=self.__destination_host, + port=self.__destination_port, + token=self.__destination_token + ) def _init_logger(self): @@ -41,6 +54,7 @@ class Migrator: self.__logger = logger + """ def _get_user_api(self, hostname, port, token) -> giteapy.UserApi: conf = giteapy.Configuration() @@ -50,7 +64,9 @@ class Migrator: api = giteapy.UserApi(giteapy.ApiClient(conf)) return api - + """ + + """ def _get_repo_api(self, hostname, port, token) -> giteapy.RepositoryApi: conf = giteapy.Configuration() @@ -60,7 +76,9 @@ class Migrator: api = giteapy.RepositoryApi(giteapy.ApiClient(conf)) return api + """ + """ def _get_org_apis(self) -> (giteapy.OrganizationApi, giteapy.OrganizationApi): api_source = self._get_org_api( @@ -73,7 +91,9 @@ class Migrator: ) return api_source, api_destination + """ + """ def _get_org_api(self, hostname, port, token) -> giteapy.OrganizationApi: conf = giteapy.Configuration() @@ -83,17 +103,9 @@ class Migrator: api = giteapy.OrganizationApi(giteapy.ApiClient(conf)) return api + """ - def _make_api_base(self, hostname, port): - - base = f"https://{hostname}" - if port is not None: - base += f":{port}" - base += self.__DEFAULT_API_PATH - - return base - - def _make_destination_repo_name(self, pattern: str, repo: giteapy.Repository): + def _make_destination_repo_name(self, pattern: str, repo: gitea.Repository): repo_name = pattern.replace(self.__REPO_ORIGINAL_NAME_TOKEN, repo.name) @@ -126,24 +138,34 @@ class Migrator: assert do_destination_copy_topics is not None, "Destination directive to copy source topics should be specified" # api_source, api_destination = self._get_org_apis() - api_source: giteapy.OrganizationApi - api_destination: giteapy.OrganizationApi + # api_source: giteapy.OrganizationApi + # api_destination: giteapy.OrganizationApi # Tattle on certify self.__logger.info(f"Certifi is currently using CA bundle: {certifi.where()}") # Grab all org repos - source_repos = self._fetch_all_org_repos(org=source_org) + source_repos = self._fetch_all_org_repos( + org_name=source_org + ) self.__logger.info(f"Found {len(source_repos)} repos on source:") for repo in source_repos: - repo: giteapy.Repository - self.__logger.info(f"- #{repo.id} {repo.full_name}") + repo: gitea.Repository + self.__logger.info(f"- {repo.get_full_name()}") print() # Filter - source_repos = self._filter_repos_for_required_topics(repos=source_repos, topics=source_topics) + source_repos = self._filter_repos_for_required_topics( + repos=source_repos, + topics_required=source_topics + ) print() + self.__logger.info(f"Have {len(source_repos)} remaining repos after topic filtering:") + for repo in source_repos: + repo: gitea.Repository + self.__logger.info(f"- {repo.get_full_name()}") + assert False, "Poop" repos_migrate = [] repos_ignore = [] @@ -239,57 +261,58 @@ class Migrator: else: self.__logger.info("Confirmation not received; Won't do anything.") - def _fetch_all_org_repos(self, org: str): + def _fetch_all_org_repos(self, org_name: str): - api_source, api_destination = self._get_org_apis() - api_source: giteapy.OrganizationApi + org = gitea.Organization.request( + gitea=self.__source_api, + name=org_name + ) - source_repos = [] + # Grabs all pages automatically + repos = org.get_repositories() - page = 0 - while True: - - page += 1 # Starts at 1 for some reason - source_repos_page = api_source.org_list_repos(org, page=page, limit=25) - - if len(source_repos_page) == 0: - break - - source_repos.extend(source_repos_page) - - return source_repos + return repos def _filter_repos_for_required_topics( self, - repos: list[giteapy.Repository], - topics: list[str] + repos: list[gitea.Repository], + topics_required: list[str] - ) -> list[giteapy.Repository]: + ) -> list[gitea.Repository]: - self.__logger.info(f"Filtering source repos for required topics: {topics}") + self.__logger.info( + f"Filtering source repos for required topics: {topics_required}" + ) repos_keep = [] repos_reject = [] repo_topics = {} - api_source_repos = self._get_repo_api( - hostname=self.__source_host, port=self.__source_port, - token=self.__source_token - ) - for repo in repos: - repo_topics[repo.id] = api_source_repos.repo_list_topics(owner=repo.owner.login, repo=repo.name) - repo_topics[repo.id] = repo_topics[repo.id].topics + repo: gitea.Repository - if self._check_required_topics(topics_present=repo_topics[repo.id], topics_required=topics): + repo_key = repo.get_full_name() + + topics_present = repo.get_topics() + print(topics_present) + repo_topics[repo_key] = topics_present["topics"] + print(repo_topics[repo_key]) + + if self._check_required_topics( + topics_present=repo_topics[repo_key], + topics_required=topics_required + ): repos_keep.append(repo) else: repos_reject.append(repo) self.__logger.info("") - self.__logger.info(f"\nKeeping {len(repos_keep)} repos because they contain all required topics ({topics}):") + self.__logger.info( + f"\nKeeping {len(repos_keep)} repos" + f" because they contain all required topics ({topics_required}):" + ) if len(repos_keep) > 0: for repo in repos_keep: self.__logger.info(f"> {repo.full_name}") @@ -297,10 +320,12 @@ class Migrator: self.__logger.info("> None") self.__logger.info("") - self.__logger.info(f"Rejecting {len(repos_reject)} repos because they don't contain all required topics:") + self.__logger.info( + f"Rejecting {len(repos_reject)} repos because they don't contain all required topics:" + ) if len(repos_reject) > 0: for repo in repos_reject: - self.__logger.info(f"> {repo.full_name} ({repo_topics[repo.id]})") + self.__logger.info(f"> {repo.full_name}") else: self.__logger.info("> None") @@ -309,6 +334,9 @@ class Migrator: @staticmethod def _check_required_topics(topics_present: list[str], topics_required: list[str]) -> bool: + print("Required topics:", topics_required) + print("Present topics:", topics_present) + for topic in topics_required: if topic not in topics_present: return False @@ -358,9 +386,9 @@ class Migrator: labels=True, issues=True, pull_requests=True, releases=True, milestones=True, wiki=True ) # TODO: These three lines represent feature request to giteapy authors - migrate_body.auth_token = self.__source_token - migrate_body.swagger_types["auth_token"] = "str" - migrate_body.attribute_map["auth_token"] = "auth_token" + #migrate_body.auth_token = self.__source_token + #migrate_body.swagger_types["auth_token"] = "str" + #migrate_body.attribute_map["auth_token"] = "auth_token" self.__logger.debug("Migrate body:") self.__logger.debug(migrate_body) @@ -416,7 +444,7 @@ class Migrator: return source_repos_successful, source_repos_failed - def _delete_migrated_repos(self, source_org_name: str, repos: list[giteapy.Repository]): + def _delete_migrated_repos(self, source_org_name: str, repos: list[gitea.Repository]): if len(repos) == 0: self.__logger.warning(f"Cannot delete any migrated repos because none were successful!")