misc-utilities/gitea-repo-migrator
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

getting a bit closer to success (import using required topics)

Browse Source
This commit is contained in:
Mike 2023-02-11 00:46:54 -08:00
parent 37426631d8
commit 9d904924a4
3 changed files with 113 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Pipfile
View File

@ -4,6 +4,7 @@ verify_ssl = true
name = "pypi"
[packages]
giteapy-soteria = {git = "https://github.com/Yousif-CS/giteapy.git"}
[dev-packages]

41
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "fedbd2ab7afd84cf16f128af0619749267b62277b4cb6989ef16d4bef6e4eef2"
"sha256": "1dc9e96fd5a12468ed7d0869b11b9fbca2464e4b806fb8b9c17391a41b6f0eb8"
},
"pipfile-spec": 6,
"requires": {
@ -15,6 +15,43 @@
}
]
},
"default": {},
"default": {
"certifi": {
"hashes": [
"sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3",
"sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"
],
"markers": "python_version >= '3.6'",
"version": "==2022.12.7"
},
"giteapy-soteria": {
"git": "https://github.com/Yousif-CS/giteapy.git",
"ref": "e0a089bdfb7ef6130b43727c50e78f176379db20"
},
"python-dateutil": {
"hashes": [
"sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
"sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
"version": "==2.8.2"
},
"six": {
"hashes": [
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
"sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
"version": "==1.16.0"
},
"urllib3": {
"hashes": [
"sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72",
"sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==1.26.14"
}
},
"develop": {}
}

92
domain/Migrator.py
View File

@ -1,7 +1,6 @@
# import giteapy
import giteapy.giteapy
import giteapy
import logging
import sys
@ -117,9 +116,8 @@ class Migrator:
api_source: giteapy.OrganizationApi
api_destination: giteapy.OrganizationApi
api_source_repos = self._get_repo_api(hostname=self.__source_host, port=self.__source_port, token=self.__source_token)
source_repos = api_source.org_list_repos(source_org, page=0, limit=1000000)
# Grab all org repos
source_repos = self._fetch_all_org_repos(org=source_org)
self.__logger.info(f"Found {len(source_repos)} repos on source:")
for repo in source_repos:
repo: giteapy.Repository
@ -127,6 +125,9 @@ class Migrator:
print()
# Filter
source_repos = self._filter_repos_for_required_topics(repos=source_repos, topics=source_topics)
repos_migrate = []
repos_ignore = []
go_right_now = False
@ -134,20 +135,6 @@ class Migrator:
repo: giteapy.Repository
repo_topics = api_source_repos.repo_list_topics(owner=repo.owner.login, repo=repo.name)
repo_topics = repo_topics.topics
self.__logger.error(f"Repo topics: {repo_topics}")
if self._check_source_repo_topics(repo=repo, topics=source_topics) is False:
repos_ignore.append(repo)
self.__logger.info(
f"Ignoring repo because it doesn't have all required tags."
f"\n> Repo: {repo.full_name}"
f"\n> Has: {repo_topics}"
f"\n> Needs: {source_topics}"
)
continue
while True:
if interactive:
@ -219,10 +206,75 @@ class Migrator:
else:
self.__logger.info("Confirmation not received; Won't do anything.")
def _check_source_repo_topics(self, repo: giteapy.Repository, topics: list[str]) -> bool:
def _fetch_all_org_repos(self, org: str):
api_source, api_destination = self._get_org_apis()
api_source: giteapy.OrganizationApi
source_repos = []
page = 0
while True:
page += 1 # Starts at 1 for some reason
source_repos_page = api_source.org_list_repos(org, page=page, limit=25)
if len(source_repos_page) == 0:
break
source_repos.extend(source_repos_page)
return source_repos
def _filter_repos_for_required_topics(self, repos: list[giteapy.Repository], topics: list[str]) -> list[giteapy.Repository]:
self.__logger.info(f"Filtering source repos for required topics: {topics}")
repos_keep = []
repos_reject = []
repo_topics = {}
api_source_repos = self._get_repo_api(
hostname=self.__source_host, port=self.__source_port,
token=self.__source_token
)
for repo in repos:
repo_topics[repo.id] = api_source_repos.repo_list_topics(owner=repo.owner.login, repo=repo.name)
repo_topics[repo.id] = repo_topics[repo.id].topics
self.__logger.error(f"Repo topics: {repo_topics[repo.id]}")
if self._check_required_topics(topics_present=repo_topics[repo.id], topics_required=topics):
repos_keep.append(repo)
else:
repos_reject.append(repo)
self.__logger.info(f"Keeping the following repos because they contain all required topics ({topics}):")
if len(repos_keep) > 0:
for repo in repos_keep:
self.__logger.info(f"> {repo.full_name}")
else:
self.__logger.info("> None")
self.__logger.info("Rejecting the following repos because they don't contain all required topics:")
if len(repos_reject) > 0:
for repo in repos_reject:
self.__logger.info(f"> {repo.full_name} ({repo_topics[repo.id]})")
else:
self.__logger.info("> None")
return repos_keep
def _check_required_topics(self, topics_present: list[str], topics_required: list[str]) -> bool:
for topic in topics_required:
if topic not in topics_present:
return False
return True
def _migrate_repos(
self,
destination_org_name: str,