From 9d904924a45eb8c5ed3b95451e2b05214f194497 Mon Sep 17 00:00:00 2001 From: Mike Date: Sat, 11 Feb 2023 00:46:54 -0800 Subject: [PATCH] getting a bit closer to success (import using required topics) --- Pipfile | 1 + Pipfile.lock | 41 +++++++++++++++++++- domain/Migrator.py | 94 +++++++++++++++++++++++++++++++++++----------- 3 files changed, 113 insertions(+), 23 deletions(-) diff --git a/Pipfile b/Pipfile index c398b0d..5d82509 100644 --- a/Pipfile +++ b/Pipfile @@ -4,6 +4,7 @@ verify_ssl = true name = "pypi" [packages] +giteapy-soteria = {git = "https://github.com/Yousif-CS/giteapy.git"} [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index eb6410c..d681723 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "fedbd2ab7afd84cf16f128af0619749267b62277b4cb6989ef16d4bef6e4eef2" + "sha256": "1dc9e96fd5a12468ed7d0869b11b9fbca2464e4b806fb8b9c17391a41b6f0eb8" }, "pipfile-spec": 6, "requires": { @@ -15,6 +15,43 @@ } ] }, - "default": {}, + "default": { + "certifi": { + "hashes": [ + "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3", + "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18" + ], + "markers": "python_version >= '3.6'", + "version": "==2022.12.7" + }, + "giteapy-soteria": { + "git": "https://github.com/Yousif-CS/giteapy.git", + "ref": "e0a089bdfb7ef6130b43727c50e78f176379db20" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.8.2" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.16.0" + }, + "urllib3": { + "hashes": [ + "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72", + "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.26.14" + } + }, "develop": {} } diff --git a/domain/Migrator.py b/domain/Migrator.py index f86432a..e62e30b 100644 --- a/domain/Migrator.py +++ b/domain/Migrator.py @@ -1,7 +1,6 @@ -# import giteapy -import giteapy.giteapy +import giteapy import logging import sys @@ -117,9 +116,8 @@ class Migrator: api_source: giteapy.OrganizationApi api_destination: giteapy.OrganizationApi - api_source_repos = self._get_repo_api(hostname=self.__source_host, port=self.__source_port, token=self.__source_token) - - source_repos = api_source.org_list_repos(source_org, page=0, limit=1000000) + # Grab all org repos + source_repos = self._fetch_all_org_repos(org=source_org) self.__logger.info(f"Found {len(source_repos)} repos on source:") for repo in source_repos: repo: giteapy.Repository @@ -127,6 +125,9 @@ class Migrator: print() + # Filter + source_repos = self._filter_repos_for_required_topics(repos=source_repos, topics=source_topics) + repos_migrate = [] repos_ignore = [] go_right_now = False @@ -134,20 +135,6 @@ class Migrator: repo: giteapy.Repository - repo_topics = api_source_repos.repo_list_topics(owner=repo.owner.login, repo=repo.name) - repo_topics = repo_topics.topics - self.__logger.error(f"Repo topics: {repo_topics}") - - if self._check_source_repo_topics(repo=repo, topics=source_topics) is False: - repos_ignore.append(repo) - self.__logger.info( - f"Ignoring repo because it doesn't have all required tags." - f"\n> Repo: {repo.full_name}" - f"\n> Has: {repo_topics}" - f"\n> Needs: {source_topics}" - ) - continue - while True: if interactive: @@ -219,9 +206,74 @@ class Migrator: else: self.__logger.info("Confirmation not received; Won't do anything.") - def _check_source_repo_topics(self, repo: giteapy.Repository, topics: list[str]) -> bool: + def _fetch_all_org_repos(self, org: str): - return False + api_source, api_destination = self._get_org_apis() + api_source: giteapy.OrganizationApi + + source_repos = [] + + page = 0 + while True: + + page += 1 # Starts at 1 for some reason + source_repos_page = api_source.org_list_repos(org, page=page, limit=25) + + if len(source_repos_page) == 0: + break + + source_repos.extend(source_repos_page) + + return source_repos + + def _filter_repos_for_required_topics(self, repos: list[giteapy.Repository], topics: list[str]) -> list[giteapy.Repository]: + + self.__logger.info(f"Filtering source repos for required topics: {topics}") + + repos_keep = [] + repos_reject = [] + repo_topics = {} + + api_source_repos = self._get_repo_api( + hostname=self.__source_host, port=self.__source_port, + token=self.__source_token + ) + + for repo in repos: + + repo_topics[repo.id] = api_source_repos.repo_list_topics(owner=repo.owner.login, repo=repo.name) + repo_topics[repo.id] = repo_topics[repo.id].topics + + self.__logger.error(f"Repo topics: {repo_topics[repo.id]}") + + if self._check_required_topics(topics_present=repo_topics[repo.id], topics_required=topics): + repos_keep.append(repo) + else: + repos_reject.append(repo) + + self.__logger.info(f"Keeping the following repos because they contain all required topics ({topics}):") + if len(repos_keep) > 0: + for repo in repos_keep: + self.__logger.info(f"> {repo.full_name}") + else: + self.__logger.info("> None") + + self.__logger.info("Rejecting the following repos because they don't contain all required topics:") + if len(repos_reject) > 0: + for repo in repos_reject: + self.__logger.info(f"> {repo.full_name} ({repo_topics[repo.id]})") + else: + self.__logger.info("> None") + + return repos_keep + + def _check_required_topics(self, topics_present: list[str], topics_required: list[str]) -> bool: + + for topic in topics_required: + if topic not in topics_present: + return False + + return True def _migrate_repos( self,