21 Commits
v1.0.0 ... dev

Author SHA1 Message Date
782729ee58 Give option to specify working directory (totally useless but too late now) 2025-01-04 02:13:40 -08:00
c43fee7024 Bump python/pipenv/pipenvlock and add init-environment script. 2025-01-04 01:02:05 -08:00
64ef546799 Point Pipfile to my forked repo (for now), and relock. 2024-06-18 02:20:52 -07:00
ec81cab79d Seems to be working now 2024-06-18 02:04:43 -07:00
27a8f73390 Work 2024-06-18 00:57:09 -07:00
17a6422b1b Started refactoring to py-gitea 2024-06-17 23:45:40 -07:00
16ab8a2ffd Track+log some migration failures instead of actually failing 2023-12-28 05:11:21 -08:00
c6e2244694 Tweak logging 2023-12-28 05:10:54 -08:00
388a0235dd Log each repo during migration 2023-12-28 04:20:42 -08:00
06b90e515c README correction 2023-12-28 03:16:34 -08:00
cb06d54d4c Bump python and deps 2023-12-28 03:08:08 -08:00
e665ba79d7 Fix SSL shenanigans with a hack to allow the user to specify the CA bundle file 2023-08-12 04:45:57 -07:00
1a6dcdeb78 typos 2023-08-12 04:45:40 -07:00
c478284764 Start using pyenv and relock pipenv 2023-08-12 04:45:15 -07:00
7fd124118f Also add ability to copy source topics 2023-02-11 01:25:21 -08:00
9c0640f2d1 Now able to filter source repos by required tags 2023-02-11 01:07:42 -08:00
9d904924a4 getting a bit closer to success (import using required topics) 2023-02-11 00:46:54 -08:00
37426631d8 uhm what 2023-02-10 23:47:22 -08:00
481e85a533 trying to upgrade the giteapy version, bleh 2023-02-10 23:46:25 -08:00
34c77b5069 Bugfix a help message 2023-02-10 06:30:03 -08:00
af61795525 quick and dirty README 2023-01-18 01:05:02 -08:00
8 changed files with 648 additions and 121 deletions

1
.python-version Normal file
View File

@ -0,0 +1 @@
3.13.1

View File

@ -4,9 +4,11 @@ verify_ssl = true
name = "pypi"
[packages]
giteapy = "*"
py-gitea = "*"
# py-gitea = {git = "https://github.com/mikeperalta1/py-gitea.git"}
[dev-packages]
[requires]
python_version = "3.10"
python_version = "3.13"

157
Pipfile.lock generated
View File

@ -1,11 +1,11 @@
{
"_meta": {
"hash": {
"sha256": "824c02127733d501ae21f3e5ccc7ee72fc4b2fe92bbca23c77ebd204d294b0fc"
"sha256": "f3059ba43523b781285862b07abfb6967aa46c609acb56be7d8ec252c75eb6d5"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.10"
"python_version": "3.13"
},
"sources": [
{
@ -18,42 +18,149 @@
"default": {
"certifi": {
"hashes": [
"sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3",
"sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"
"sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56",
"sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"
],
"markers": "python_version >= '3.6'",
"version": "==2022.12.7"
"version": "==2024.12.14"
},
"giteapy": {
"charset-normalizer": {
"hashes": [
"sha256:2078c802a4626bf311e911c969c34b7d19fbe9175e2910e1965b24ff69221470"
"sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537",
"sha256:01732659ba9b5b873fc117534143e4feefecf3b2078b0a6a2e925271bb6f4cfa",
"sha256:01ad647cdd609225c5350561d084b42ddf732f4eeefe6e678765636791e78b9a",
"sha256:04432ad9479fa40ec0f387795ddad4437a2b50417c69fa275e212933519ff294",
"sha256:0907f11d019260cdc3f94fbdb23ff9125f6b5d1039b76003b5b0ac9d6a6c9d5b",
"sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd",
"sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601",
"sha256:09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd",
"sha256:0af291f4fe114be0280cdd29d533696a77b5b49cfde5467176ecab32353395c4",
"sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d",
"sha256:1a2bc9f351a75ef49d664206d51f8e5ede9da246602dc2d2726837620ea034b2",
"sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313",
"sha256:234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd",
"sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa",
"sha256:237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8",
"sha256:28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1",
"sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2",
"sha256:2a75d49014d118e4198bcee5ee0a6f25856b29b12dbf7cd012791f8a6cc5c496",
"sha256:2bdfe3ac2e1bbe5b59a1a63721eb3b95fc9b6817ae4a46debbb4e11f6232428d",
"sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b",
"sha256:2fb9bd477fdea8684f78791a6de97a953c51831ee2981f8e4f583ff3b9d9687e",
"sha256:311f30128d7d333eebd7896965bfcfbd0065f1716ec92bd5638d7748eb6f936a",
"sha256:329ce159e82018d646c7ac45b01a430369d526569ec08516081727a20e9e4af4",
"sha256:345b0426edd4e18138d6528aed636de7a9ed169b4aaf9d61a8c19e39d26838ca",
"sha256:363e2f92b0f0174b2f8238240a1a30142e3db7b957a5dd5689b0e75fb717cc78",
"sha256:3a3bd0dcd373514dcec91c411ddb9632c0d7d92aed7093b8c3bbb6d69ca74408",
"sha256:3bed14e9c89dcb10e8f3a29f9ccac4955aebe93c71ae803af79265c9ca5644c5",
"sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3",
"sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f",
"sha256:4532bff1b8421fd0a320463030c7520f56a79c9024a4e88f01c537316019005a",
"sha256:49402233c892a461407c512a19435d1ce275543138294f7ef013f0b63d5d3765",
"sha256:4c0907b1928a36d5a998d72d64d8eaa7244989f7aaaf947500d3a800c83a3fd6",
"sha256:4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146",
"sha256:5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6",
"sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9",
"sha256:619a609aa74ae43d90ed2e89bdd784765de0a25ca761b93e196d938b8fd1dbbd",
"sha256:6e27f48bcd0957c6d4cb9d6fa6b61d192d0b13d5ef563e5f2ae35feafc0d179c",
"sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f",
"sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545",
"sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176",
"sha256:75832c08354f595c760a804588b9357d34ec00ba1c940c15e31e96d902093770",
"sha256:7709f51f5f7c853f0fb938bcd3bc59cdfdc5203635ffd18bf354f6967ea0f824",
"sha256:78baa6d91634dfb69ec52a463534bc0df05dbd546209b79a3880a34487f4b84f",
"sha256:7974a0b5ecd505609e3b19742b60cee7aa2aa2fb3151bc917e6e2646d7667dcf",
"sha256:7a4f97a081603d2050bfaffdefa5b02a9ec823f8348a572e39032caa8404a487",
"sha256:7b1bef6280950ee6c177b326508f86cad7ad4dff12454483b51d8b7d673a2c5d",
"sha256:7d053096f67cd1241601111b698f5cad775f97ab25d81567d3f59219b5f1adbd",
"sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b",
"sha256:807f52c1f798eef6cf26beb819eeb8819b1622ddfeef9d0977a8502d4db6d534",
"sha256:80ed5e856eb7f30115aaf94e4a08114ccc8813e6ed1b5efa74f9f82e8509858f",
"sha256:8417cb1f36cc0bc7eaba8ccb0e04d55f0ee52df06df3ad55259b9a323555fc8b",
"sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9",
"sha256:89149166622f4db9b4b6a449256291dc87a99ee53151c74cbd82a53c8c2f6ccd",
"sha256:8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125",
"sha256:8c60ca7339acd497a55b0ea5d506b2a2612afb2826560416f6894e8b5770d4a9",
"sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de",
"sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11",
"sha256:97f68b8d6831127e4787ad15e6757232e14e12060bec17091b85eb1486b91d8d",
"sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35",
"sha256:9f0b8b1c6d84c8034a44893aba5e767bf9c7a211e313a9605d9c617d7083829f",
"sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda",
"sha256:ab36c8eb7e454e34e60eb55ca5d241a5d18b2c6244f6827a30e451c42410b5f7",
"sha256:b010a7a4fd316c3c484d482922d13044979e78d1861f0e0650423144c616a46a",
"sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971",
"sha256:b7b2d86dd06bfc2ade3312a83a5c364c7ec2e3498f8734282c6c3d4b07b346b8",
"sha256:b97e690a2118911e39b4042088092771b4ae3fc3aa86518f84b8cf6888dbdb41",
"sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d",
"sha256:c0429126cf75e16c4f0ad00ee0eae4242dc652290f940152ca8c75c3a4b6ee8f",
"sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757",
"sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a",
"sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886",
"sha256:c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77",
"sha256:d7f50a1f8c450f3925cb367d011448c39239bb3eb4117c36a6d354794de4ce76",
"sha256:d973f03c0cb71c5ed99037b870f2be986c3c05e63622c017ea9816881d2dd247",
"sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85",
"sha256:d9c3cdf5390dcd29aa8056d13e8e99526cda0305acc038b96b30352aff5ff2bb",
"sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7",
"sha256:dccbe65bd2f7f7ec22c4ff99ed56faa1e9f785482b9bbd7c717e26fd723a1d1e",
"sha256:dd78cfcda14a1ef52584dbb008f7ac81c1328c0f58184bf9a84c49c605002da6",
"sha256:e218488cd232553829be0664c2292d3af2eeeb94b32bea483cf79ac6a694e037",
"sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1",
"sha256:ea0d8d539afa5eb2728aa1932a988a9a7af94f18582ffae4bc10b3fbdad0626e",
"sha256:eab677309cdb30d047996b36d34caeda1dc91149e4fdca0b1a039b3f79d9a807",
"sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407",
"sha256:ecddf25bee22fe4fe3737a399d0d177d72bc22be6913acfab364b40bce1ba83c",
"sha256:eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12",
"sha256:f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3",
"sha256:f30bf9fd9be89ecb2360c7d94a711f00c09b976258846efe40db3d05828e8089",
"sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd",
"sha256:fc54db6c8593ef7d4b2a331b58653356cf04f67c960f584edb7c3d8c97e8f39e",
"sha256:fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00",
"sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616"
],
"markers": "python_version >= '3.7'",
"version": "==3.4.1"
},
"idna": {
"hashes": [
"sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9",
"sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"
],
"markers": "python_version >= '3.6'",
"version": "==3.10"
},
"immutabledict": {
"hashes": [
"sha256:c56a26ced38c236f79e74af3ccce53772827cef5c3bce7cab33ff2060f756373",
"sha256:d91017248981c72eb66c8ff9834e99c2f53562346f23e7f51e7a5ebcf66a3bcc"
],
"markers": "python_version >= '3.8'",
"version": "==4.2.1"
},
"py-gitea": {
"hashes": [
"sha256:aa9433cb83a528a8560de7affc0d85a838de17a7b28b43be5f0066341af4fda8",
"sha256:f7641ea0818529b59f3ca9c38b106a5c2ba06a0f662a022f76a62919f17e8379"
],
"index": "pypi",
"version": "==1.0.8"
"version": "==0.2.8"
},
"python-dateutil": {
"requests": {
"hashes": [
"sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
"sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
"sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760",
"sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==2.8.2"
},
"six": {
"hashes": [
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
"sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==1.16.0"
"markers": "python_version >= '3.8'",
"version": "==2.32.3"
},
"urllib3": {
"hashes": [
"sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72",
"sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"
"sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df",
"sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==1.26.14"
"markers": "python_version >= '3.9'",
"version": "==2.3.0"
}
},
"develop": {}

View File

@ -1,2 +1,57 @@
# gitea-repo-migrator
# Mike's Gitea Repo Migrator
Just a script to help make it a little easier to migrate an entire organization (with bulk selection) from one Gitea instance to another.
Supports changing destination names a bit, and adding topics to each transferred repo, as well as optional bulk-selected deletion from the source.
by Mike Peralta
Current license: You are free to clone and use this program but all other rights reserved, provided you accept 100% of all liability of any outcome of use/download/etc. this program. Better license coming soon.
## Requirements
* python 3.12
* pipenv
## Installation
1. Clone this repo and `cd` to its root directory
2. Execute `pipenv install` (and optionally check with `pipenv check`)
## Execution
Ask `pipenv` to execute *main.py* on your behalf with the following:
```console
$ cd /path/to/gitea-repo-migrator
$ pipenv run python ./main.py --help
```
Optionally ask for a shell to run multiple times a bit faster:
```console
$ cd /path/to/gitea-repo-migrator
$ pipenv shell
$ python ./main.py --help
$ python ./main.py --help
```
### App Tokens
You'll need to generate an *Application Token* for both the source and destination servers, and pass the token along to the command line.
### SSL Verification
Pass the long switch `--no-verify-ssl` if any server sits behind a self-signed or wonky SSL certificate.
### Destination Repo Names
You can tweak the destination repo names a bit by using a string that includes `%N%` somewhere. The string `%N%` will expand to the original name. For example, if we use a repo originally named `my-test-repo` with the string `what-%N%`, the resulting destination repo name will be `what-my-test-repo`. This string will be recomputed for every migrated repo.
### Destination Topics
Topics will be duplicated from all source repos to their corresponding destination repos. You can specify additional topics with the `--destination-topic` switch. For example, to add the topic `migrated` to every repo, pass the switch `--destination-topic migrated`.

43
domain/API.py Normal file
View File

@ -0,0 +1,43 @@
import gitea
class API:
__DEFAULT_API_PATH = "/api/v1"
def __init__(self, verify_ssl, ca_bundle):
self.__verify_ssl = verify_ssl
self.__ca_bundle = ca_bundle
@staticmethod
def _make_api_base_url(hostname, port):
base = f"https://{hostname}"
if port is not None:
base += f":{port}"
return base
def get(self, hostname, port, token) -> gitea.Gitea:
url = API._make_api_base_url(
hostname=hostname,
port=port
)
ssl_verify_arg = True
if self.__verify_ssl is not None:
ssl_verify_arg = self.__verify_ssl
if self.__ca_bundle is not None:
ssl_verify_arg = self.__ca_bundle
g = gitea.Gitea(
gitea_url=url,
token_text=token,
verify=ssl_verify_arg
)
return g

View File

@ -1,27 +1,30 @@
import giteapy
from domain.API import API
import gitea
import logging
import sys
import certifi
class Migrator:
__DEFAULT_API_PATH = "/api/v1"
__REPO_ORIGINAL_NAME_TOKEN = "%N%"
def __init__(
self,
source_host, source_port, source_token,
destination_host, destination_port, destination_token,
verify_ssl: bool = True, ca_bundle: str = None
):
# noinspection PyTypeChecker
self.__logger: logging.Logger = None
self._init_logger()
self.__verify_ssl = True
self.__source_host = source_host
self.__source_port = source_port
self.__source_token = source_token
@ -29,6 +32,26 @@ class Migrator:
self.__destination_host = destination_host
self.__destination_port = destination_port
self.__destination_token = destination_token
self.__verify_ssl = verify_ssl
self.__ca_bundle = ca_bundle
api = API(
verify_ssl=self.__verify_ssl,
ca_bundle=self.__ca_bundle,
)
self.__source_api = api.get(
hostname=self.__source_host,
port=self.__source_port,
token=self.__source_token,
)
self.__destination_api = api.get(
hostname=self.__destination_host,
port=self.__destination_port,
token=self.__destination_token,
)
def _init_logger(self):
@ -39,6 +62,7 @@ class Migrator:
self.__logger = logger
"""
def _get_user_api(self, hostname, port, token) -> giteapy.UserApi:
conf = giteapy.Configuration()
@ -48,7 +72,9 @@ class Migrator:
api = giteapy.UserApi(giteapy.ApiClient(conf))
return api
"""
"""
def _get_repo_api(self, hostname, port, token) -> giteapy.RepositoryApi:
conf = giteapy.Configuration()
@ -58,7 +84,9 @@ class Migrator:
api = giteapy.RepositoryApi(giteapy.ApiClient(conf))
return api
"""
"""
def _get_org_apis(self) -> (giteapy.OrganizationApi, giteapy.OrganizationApi):
api_source = self._get_org_api(
@ -71,7 +99,9 @@ class Migrator:
)
return api_source, api_destination
"""
"""
def _get_org_api(self, hostname, port, token) -> giteapy.OrganizationApi:
conf = giteapy.Configuration()
@ -81,53 +111,87 @@ class Migrator:
api = giteapy.OrganizationApi(giteapy.ApiClient(conf))
return api
"""
def _make_api_base(self, hostname, port):
base = f"https://{hostname}"
if port is not None:
base += f":{port}"
base += self.__DEFAULT_API_PATH
return base
def _make_destination_repo_name(self, pattern: str, repo: giteapy.Repository):
def _make_destination_repo_name(self, pattern: str, repo: gitea.Repository):
repo_name = pattern.replace(self.__REPO_ORIGINAL_NAME_TOKEN, repo.name)
return repo_name
def set_verify_ssl(self, b: bool):
"""
def set_ca_bundle(self, bundle_path: str):
self.__verify_ssl = b
self.__logger.info("Setting certificate bundle path")
# Hacky but oh well
self.__logger.info(f"Old path: {certifi.where()}")
certifi.core._CACERT_PATH = bundle_path
self.__logger.info(f"New path: {certifi.where()}")
# TODO: JUST TESTING
self.__verify_ssl = bundle_path
"""
def migrate_entire_org(
self,
source_org: str,
destination_org: str, destination_repo_name: str, destination_topics: list
self,
interactive: bool = True,
source_org: str = None, source_topics: list[str] = None,
destination_org: str = None, destination_repo_name: str = None, destination_topics: list = None,
do_destination_copy_topics: bool = True
):
assert source_org is not None, "Source org must be specified"
assert destination_org is not None, "Destination org must be specified"
assert destination_repo_name is not None, "Destination repo name must be specified"
assert destination_topics is not None, "Destination topics must be specified"
assert do_destination_copy_topics is not None, "Destination directive to copy source topics should be specified"
api_source, api_destination = self._get_org_apis()
# api_source, api_destination = self._get_org_apis()
# api_source: giteapy.OrganizationApi
# api_destination: giteapy.OrganizationApi
source_repos = api_source.org_list_repos(source_org)
# Tattle on certify
self.__logger.info(f"Certifi is currently using CA bundle: {certifi.where()}")
# Grab all org repos
source_repos = self._fetch_all_org_repos(
org_name=source_org
)
self.__logger.info(f"Found {len(source_repos)} repos on source:")
for repo in source_repos:
repo: giteapy.Repository
self.__logger.info(f"- #{repo.id} {repo.full_name}")
repo: gitea.Repository
self.__logger.info(f"- {repo.get_full_name()}")
print()
# Filter
source_repos = self._filter_repos_for_required_topics(
repos=source_repos,
topics_required=source_topics
)
print()
self.__logger.info(f"Have {len(source_repos)} remaining repos after topic filtering:")
for repo in source_repos:
repo: gitea.Repository
self.__logger.info(f"- {repo.get_full_name()}")
repos_migrate = []
repos_ignore = []
go_right_now = False
for repo in source_repos:
repo: giteapy.Repository
repo: gitea.Repository
while True:
response = input(f"Migrate repo #{repo.id} \"{repo.full_name}\" ? (Y)es, (N)o, (G)o right now, (Q)uit ==> ")
response = response.lower()
if interactive:
response = input(
f"Migrate repo #{repo.id} \"{repo.full_name}\" ?"
" (Y)es, (N)o, (G)o right now, (Q)uit ==> "
)
response = response.lower()
else:
response = "y"
valid_input = True
if response == "y":
@ -152,26 +216,32 @@ class Migrator:
if go_right_now:
break
#
# Announce repo destination names
self.__logger.info("")
if len(repos_migrate):
self.__logger.info("Repos to migrate:")
for repo in repos_migrate:
repo: giteapy.Repository
destination_name = self._make_destination_repo_name(pattern=destination_repo_name, repo=repo)
self.__logger.info(f"#{repo.id} \"{repo.name}\" ==> \"{destination_name}\"")
repo: gitea.Repository
destination_name = self._make_destination_repo_name(
pattern=destination_repo_name, repo=repo
)
self.__logger.info(
f"#{repo.id} \"{repo.name}\"\n> \"{destination_name}\""
)
else:
self.__logger.info("No repos marked to migrate")
# Announce manually ignored repos
self.__logger.info("")
if len(repos_ignore):
self.__logger.info("Repos to ignore:")
for repo in repos_ignore:
repo: giteapy.Repository
self.__logger.info(f"#{repo.id} \"{repo.name}\"")
repo: gitea.Repository
self.__logger.info(f"#{repo.id} \"{repo.get_full_name()}\"")
else:
self.__logger.info("No repos marked to ignore")
# Migrate
if len(repos_migrate):
confirmation = input("Do you confirm the above selections? Enter MIGRATE ==> ")
@ -179,98 +249,231 @@ class Migrator:
if confirmation == "MIGRATE":
self.__logger.info("Confirmation received; Processing ... ")
source_repos_successful = self._migrate_repos(
source_repos_successful, source_repos_failed = self._migrate_repos(
destination_org_name=destination_org,
destination_repo_name=destination_repo_name,
destination_topics=destination_topics,
do_destination_copy_topics=do_destination_copy_topics,
repos=repos_migrate
)
self.__logger.info(f"{len(source_repos_successful)} of {len(repos_migrate)} repos successfully migrated.")
self.__logger.info(
f"{len(source_repos_successful)} of {len(repos_migrate)}"
" repos successfully migrated."
)
if len(source_repos_failed) > 0:
self.__logger.error(f"Failed to migrate {len(source_repos_failed)} repos:")
for repo, exception in source_repos_failed:
self.__logger.error(
f"> {repo.name}"
)
self.__logger.error(f"Captured exception data:")
for repo, exception in source_repos_failed:
self.__logger.error(
f"Failed to migrate repo: {repo.name}\n> {exception}"
)
self._delete_migrated_repos(source_org_name=source_org, repos=source_repos_successful)
self._delete_migrated_repos(
source_org_name=source_org,
repos=source_repos_successful
)
else:
self.__logger.info("Confirmation not received; Won't do anything.")
def _fetch_all_org_repos(self, org_name: str):
org = gitea.Organization.request(
gitea=self.__source_api,
name=org_name
)
# Grabs all pages automatically
repos = org.get_repositories()
return repos
def _filter_repos_for_required_topics(
self,
repos: list[gitea.Repository],
topics_required: list[str]
) -> list[gitea.Repository]:
self.__logger.info(
f"Filtering source repos for required topics: {topics_required}"
)
repos_keep = []
repos_reject = []
repo_topics = {}
for repo in repos:
repo: gitea.Repository
repo_key = repo.get_full_name()
topics_present = repo.get_topics()
repo_topics[repo_key] = topics_present
if self._check_required_topics(
topics_present=repo_topics[repo_key],
topics_required=topics_required
):
repos_keep.append(repo)
else:
repos_reject.append(repo)
self.__logger.info("")
self.__logger.info(
f"\nKeeping {len(repos_keep)} repos"
f" because they contain all required topics ({topics_required}):"
)
if len(repos_keep) > 0:
for repo in repos_keep:
self.__logger.info(f"> {repo.full_name}")
else:
self.__logger.info("> None")
self.__logger.info("")
self.__logger.info(
f"Rejecting {len(repos_reject)} repos because they don't contain all required topics:"
)
if len(repos_reject) > 0:
for repo in repos_reject:
self.__logger.info(f"> {repo.full_name}")
else:
self.__logger.info("> None")
return repos_keep
@staticmethod
def _check_required_topics(topics_present: list[str], topics_required: list[str]) -> bool:
for topic in topics_required:
if topic not in topics_present:
return False
return True
def _migrate_repos(
self,
destination_org_name: str,
destination_repo_name: str,
destination_topics: list,
do_destination_copy_topics: bool,
repos: list
):
api_source, api_destination = self._get_org_apis()
# api_source, api_destination = self._get_org_apis()
# destination_org = api_destination.org_get(org=destination_org_name)
# destination_org: giteapy.Organization
destination_org = api_destination.org_get(org=destination_org_name)
destination_org: giteapy.Organization
api_dest_org = gitea.Organization.request(
gitea=self.__destination_api,
name=destination_org_name
)
self.__logger.info(f"Destination organization: {destination_org.full_name}")
self.__logger.info(f"Destination organization: {api_dest_org.full_name}")
source_repos_successful = []
source_repos_failed = []
for source_repo in repos:
source_repo: giteapy.Repository
source_repo: gitea.Repository
this_destination_repo_name = destination_repo_name.replace("%N%", source_repo.name)
migrate_body = giteapy.MigrateRepoForm(
mirror=False,
clone_addr=source_repo.clone_url,
uid=destination_org.id,
private=source_repo.private,
repo_name=this_destination_repo_name,
description=source_repo.description,
labels=True, issues=True, pull_requests=True, releases=True, milestones=True, wiki=True
this_destination_repo_name = destination_repo_name.replace(
"%N%",
source_repo.name
)
# TODO: These three lines represent feature request to giteapy authors
migrate_body.auth_token = self.__source_token
migrate_body.swagger_types["auth_token"] = "str"
migrate_body.attribute_map["auth_token"] = "auth_token"
self.__logger.debug("Migrate body:")
self.__logger.debug(migrate_body)
destination_api = self._get_repo_api(
hostname=self.__destination_host,
port=self.__destination_port,
token=self.__destination_token,
self.__logger.info(
f"Migrating: {source_repo.name} ==> {this_destination_repo_name}"
)
repo_new = destination_api.repo_migrate(body=migrate_body)
source_repo_topics = source_repo.get_topics()
try:
repo_new = gitea.Repository.migrate_repo(
gitea=self.__destination_api,
service="gitea", # type of remote service
clone_addr=source_repo.clone_url,
repo_name=this_destination_repo_name,
description=source_repo.description,
private=source_repo.private,
auth_token=self.__source_token,
auth_username=None,
auth_password=None,
mirror=False,
mirror_interval=None,
# lfs=False,
# lfs_endpoint="",
wiki=True,
labels=True,
issues=True,
pull_requests=True,
releases=True,
milestones=True,
repo_owner=destination_org_name,
)
except Exception as e:
self.__logger.error(
f"Failed to execute repo migration request:"
f"\n{e}"
)
source_repos_failed.append(
(source_repo, e)
)
continue
self.__logger.debug(f"Migration result: {repo_new}")
repo_new: giteapy.Repository
repo_new: gitea.Repository
assert repo_new.name == this_destination_repo_name,\
assert repo_new.name == this_destination_repo_name, \
"New repository didn't end up with the correct name. Failure?"
# Copy source topics?
if do_destination_copy_topics:
for topic in source_repo_topics:
self.__logger.debug(f"Appending source topic to new repo: {topic}")
repo_new.add_topic(topic=topic)
# Add specified topics
for topic in destination_topics:
self.__logger.debug(f"Appending topic to new repo: {topic}")
destination_api.repo_add_topc(
owner=destination_org.username,
repo=repo_new.name,
topic=topic,
)
repo_new.add_topic(topic=topic)
source_repos_successful.append(source_repo)
return source_repos_successful
return source_repos_successful, source_repos_failed
def _delete_migrated_repos(self, source_org_name: str, repos: list[giteapy.Repository]):
def _delete_migrated_repos(self, source_org_name: str, repos: list[gitea.Repository]):
repo_api = self._get_repo_api(
hostname=self.__source_host,
port=self.__source_port,
token=self.__source_token
)
repo_api: giteapy.RepositoryApi
if len(repos) == 0:
self.__logger.warning(f"Cannot delete any migrated repos because none were successful!")
return
self.__logger.info("")
self.__logger.info("Can now delete the following successfully migrated repos:")
self.__logger.info(f"Can now delete repos from source org: {source_org_name}")
self.__logger.info(f"Will delete {len(repos)} successfully migrated repos:")
for r in repos:
r: gitea.Repository
self.__logger.info(f"> #{r.id} \"{r.full_name}\" ==> {r.clone_url}")
response = input("Would you like to delete the successfully migrated repos? Type DELETE ==> ")
# Ask the user to confirm deletion
response = input(
"Would you like to delete the successfully migrated repos? Type DELETE ==> "
)
if response != "DELETE":
self.__logger.info("Okay, won't delete migrated repos.")
return
@ -279,7 +482,9 @@ class Migrator:
do_delete_all = False
for repo in repos:
self.__logger.info(f"Next repo to delete: #{repo.id} \"{repo.full_name}\"")
repo: gitea.Repository
self.__logger.info(f"Next repo to delete: \"{repo.full_name}\"")
do_delete = True if do_delete_all else False
if do_delete is False:
@ -313,7 +518,4 @@ class Migrator:
self.__logger.info(f"Deleting repo: {repo.full_name}")
repo_api.repo_delete(
owner=source_org_name,
repo=repo.name
)
repo.delete()

54
init-environment Executable file
View File

@ -0,0 +1,54 @@
#!/bin/bash
log()
{
echo "[Mike's Gitea Repo Migrator - Init Env] $1"
}
complain()
{
echo "[Mike's Gitea Repo Migrator - Init Env] $1" 1>&2
}
die()
{
complain "Fatal: $1"
exit 1
}
SCRIPT_PATH=$(readlink -f "$0")
SCRIPT_DIR=$(dirname "$SCRIPT_PATH")
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
log "Begin ${SCRIPT_NAME}"
log "Script path: ${SCRIPT_PATH}"
log "Script dir: ${SCRIPT_DIR}"
log "PATH: ${PATH}"
log "PWD before switching: $(pwd)"
cd "${SCRIPT_DIR}" || die "Failed to switch to project directory: ${SCRIPT_DIR}"
log "PWD after switching: $(pwd)"
log "Printing environment:"
printenv
log "Ensuring python installation with pyenv"
pyenv versions
pyenv install --skip-existing || die "Failed to ensure python installation with pyenv"
log "Installing/upgrading pip and pipenv"
pip install --upgrade pip pipenv || die "Failed to install/upgrade pip and pipenv"
#log "Removing old pip environment"
#pipenv --rm # Don't die because this will return an error if the env didn't already exist
# Delete and relock, because frikkin nvidia driver deps are inconsistent between x86_64 and RPi
#log "Re-locking pip dependencies"
#rm Pipfile.lock
#pipenv lock || die "Unable to pipenv lock !"
# Actually install/sync
log "Syncing pip dependencies"
pipenv sync || die "Failed to sync pip environment with pipenv"

77
main.py
View File

@ -4,6 +4,7 @@ from domain.Migrator import Migrator
import argparse
import os
def main():
@ -12,6 +13,14 @@ def main():
prog="Mike's Gitea Repo Migrator - Move repositories from one Gitea instance to another"
)
parser.add_argument(
"--cwd", "--working-directory",
dest="working_directory",
required=False,
default=None,
help="Specify the working directory"
)
parser.add_argument(
"--source-hostname", "--source-host",
dest="source_hostname",
@ -23,7 +32,7 @@ def main():
dest="source_port",
required=False,
default=None,
help="Port of the source server"
help="Port of the source server. Requests will use https (not ssh), so you probably don't want to change this."
)
parser.add_argument(
"--source-token",
@ -37,6 +46,14 @@ def main():
required=True,
help="Name of the source organization"
)
parser.add_argument(
"--source-required-topic", "--source-topic",
dest="source_topics",
default=[],
action="append",
help="Specify zero or more topics required topics for source repositories."
" Any repository that doesn't have all required topics will be skipped"
)
parser.add_argument(
"--destination-hostname", "--dest-hostname", "--destination-host", "--dest-host",
@ -49,7 +66,7 @@ def main():
dest="destination_port",
required=False,
default=None,
help="Port of the destination server"
help="Port of the destination server. Requests will use https (not ssh), so you probably don't want to change this."
)
parser.add_argument(
"--destination-token", "--dest-token",
@ -68,16 +85,46 @@ def main():
"--destination-repo-name", "--destination-name", "--dest-repo-name", "--dest-name",
dest="destination_repo_name",
default="%N%",
help="Specify the destination repository name(s). Use wildcard %N% anywhere to denote the original name"
help="Specify the destination repository name(s). Use wildcard %%N%% anywhere to denote the original name"
)
parser.add_argument(
"--destination-add-topic", "--destination-topic", "-dest-add-topic", "--dest-topic",
"--destination-add-topic", "--destination-topic", "--dest-add-topic", "--dest-topic",
dest="destination_topics",
default=[],
action="append",
help="Specify zero or more topics to add to each destination repository"
)
parser.add_argument(
"--destination-copy-topics", "--dest-copy-topics",
dest="do_destination_copy_topics",
default=True,
action="store_true",
help="Destination repos should copy topics from their source."
)
parser.add_argument(
"--no-destination-copy-topics", "--no-dest-copy-topics",
dest="do_destination_copy_topics",
default=True,
action="store_false",
help="Destination repos should NOT copy topics from their source."
)
parser.add_argument(
"--interactive",
dest="interactive",
default=True,
action="store_true",
help="Ask to confirm each migration.",
)
parser.add_argument(
"--no-interactive", "--non-interactive",
dest="interactive",
default=True,
action="store_false",
help="Do not ask to confirm each migration; Migrate all repos quickly.",
)
parser.add_argument(
"--no-verify-ssl",
dest="verify_ssl",
@ -86,21 +133,37 @@ def main():
help="Don't verify SSL certificates",
)
parser.add_argument(
"--ca-bundle",
dest="ca_bundle",
default=None,
help="Specify the location of your system-wide CA Bundle, in case python is not using it."
)
args = parser.parse_args()
if args.working_directory is not None:
os.chdir(args.working_directory)
mig = Migrator(
source_host=args.source_hostname,
source_port=args.source_port,
source_token=args.source_token,
destination_host=args.destination_hostname,
destination_port=args.destination_port,
destination_token=args.destination_token
destination_token=args.destination_token,
verify_ssl=args.verify_ssl,
ca_bundle=args.ca_bundle
)
mig.set_verify_ssl(args.verify_ssl)
mig.migrate_entire_org(
interactive=args.interactive,
source_org=args.source_org,
source_topics=args.source_topics,
destination_org=args.destination_org,
destination_repo_name=args.destination_repo_name,
destination_topics=args.destination_topics
destination_topics=args.destination_topics,
do_destination_copy_topics=args.do_destination_copy_topics
)