Compare commits
24 Commits
Author | SHA1 | Date | |
---|---|---|---|
1517f64424 | |||
cf3bd8eb85 | |||
2ef7aaf195 | |||
3c4ed5f792 | |||
403531d7f0 | |||
bd088ac545 | |||
d2af261477 | |||
5828a0363f | |||
b2261a3c49 | |||
6ceaa89dad | |||
2f706bacf6 | |||
dbbe3b88af | |||
b8aff6429f | |||
33c4233797 | |||
1564f4cf8a | |||
2e6d268de0 | |||
bfafd890d0 | |||
84791c85a7 | |||
cd14dc6158 | |||
e4d5e1b595 | |||
2e4638e448 | |||
916064c153 | |||
ec894014c6 | |||
d66688eb3c |
@ -1 +1 @@
|
|||||||
3.10.10
|
3.12.4
|
||||||
|
5
Pipfile
5
Pipfile
@ -4,11 +4,12 @@ verify_ssl = true
|
|||||||
name = "pypi"
|
name = "pypi"
|
||||||
|
|
||||||
[packages]
|
[packages]
|
||||||
pyyaml = ">=5.4"
|
#pyyaml = ">=5.4"
|
||||||
|
pyyaml = "*"
|
||||||
|
|
||||||
[dev-packages]
|
[dev-packages]
|
||||||
|
|
||||||
[requires]
|
[requires]
|
||||||
python_version = "3.10.10"
|
python_version = "3.12"
|
||||||
|
|
||||||
|
|
||||||
|
91
Pipfile.lock
generated
91
Pipfile.lock
generated
@ -1,11 +1,11 @@
|
|||||||
{
|
{
|
||||||
"_meta": {
|
"_meta": {
|
||||||
"hash": {
|
"hash": {
|
||||||
"sha256": "cceb18d3baeb19edef3ba31b743720003102c4c3d9cddd6b595c664692a37384"
|
"sha256": "32165f2eba4fd1d8db46f280f7cc3f2c9a5ed5ed87eb9ecc34f26b08ffdee5ac"
|
||||||
},
|
},
|
||||||
"pipfile-spec": 6,
|
"pipfile-spec": 6,
|
||||||
"requires": {
|
"requires": {
|
||||||
"python_version": "3.10.5"
|
"python_version": "3.12"
|
||||||
},
|
},
|
||||||
"sources": [
|
"sources": [
|
||||||
{
|
{
|
||||||
@ -18,42 +18,61 @@
|
|||||||
"default": {
|
"default": {
|
||||||
"pyyaml": {
|
"pyyaml": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293",
|
"sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5",
|
||||||
"sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b",
|
"sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc",
|
||||||
"sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57",
|
"sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df",
|
||||||
"sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b",
|
"sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741",
|
||||||
"sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4",
|
"sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206",
|
||||||
"sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07",
|
"sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27",
|
||||||
"sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba",
|
"sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595",
|
||||||
"sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9",
|
"sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62",
|
||||||
"sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287",
|
"sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98",
|
||||||
"sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513",
|
"sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696",
|
||||||
"sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0",
|
"sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290",
|
||||||
"sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0",
|
"sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9",
|
||||||
"sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92",
|
"sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d",
|
||||||
"sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f",
|
"sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6",
|
||||||
"sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2",
|
"sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867",
|
||||||
"sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc",
|
"sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47",
|
||||||
"sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c",
|
"sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486",
|
||||||
"sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86",
|
"sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6",
|
||||||
"sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4",
|
"sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3",
|
||||||
"sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c",
|
"sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007",
|
||||||
"sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34",
|
"sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938",
|
||||||
"sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b",
|
"sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0",
|
||||||
"sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c",
|
"sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c",
|
||||||
"sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb",
|
"sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735",
|
||||||
"sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737",
|
"sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d",
|
||||||
"sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3",
|
"sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28",
|
||||||
"sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d",
|
"sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4",
|
||||||
"sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53",
|
"sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
|
||||||
"sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78",
|
"sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
|
||||||
"sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803",
|
"sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef",
|
||||||
"sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a",
|
"sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
|
||||||
"sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174",
|
"sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd",
|
||||||
"sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"
|
"sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3",
|
||||||
|
"sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0",
|
||||||
|
"sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515",
|
||||||
|
"sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c",
|
||||||
|
"sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c",
|
||||||
|
"sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924",
|
||||||
|
"sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34",
|
||||||
|
"sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43",
|
||||||
|
"sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859",
|
||||||
|
"sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673",
|
||||||
|
"sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54",
|
||||||
|
"sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a",
|
||||||
|
"sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b",
|
||||||
|
"sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab",
|
||||||
|
"sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa",
|
||||||
|
"sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c",
|
||||||
|
"sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585",
|
||||||
|
"sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d",
|
||||||
|
"sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"
|
||||||
],
|
],
|
||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==6.0"
|
"markers": "python_version >= '3.6'",
|
||||||
|
"version": "==6.0.1"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"develop": {}
|
"develop": {}
|
||||||
|
@ -6,56 +6,75 @@ Mike's Backup Rotator
|
|||||||
|
|
||||||
A simple script to help automatically rotate backup files
|
A simple script to help automatically rotate backup files
|
||||||
|
|
||||||
Copyright 2023 Mike Peralta; All rights reserved
|
Copyright 2024 Mike Peralta; All rights reserved
|
||||||
|
|
||||||
Releasing to the public under the GNU GENERAL PUBLIC LICENSE v3 (See LICENSE file for more)
|
Releasing to the public under the GNU GENERAL PUBLIC LICENSE v3 (See LICENSE file for more)
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
from domain.config.Config import Config
|
||||||
|
from domain.config.ConfigFile import ConfigFile
|
||||||
from domain.Logger import Logger
|
from domain.Logger import Logger
|
||||||
from domain.Config import Config
|
from domain.Util import Util
|
||||||
|
|
||||||
|
|
||||||
import datetime
|
import datetime
|
||||||
import os
|
from pathlib import Path
|
||||||
# import pprint
|
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
|
|
||||||
class BackupRotator:
|
class BackupRotator:
|
||||||
|
|
||||||
def __init__(self, debug:bool = False):
|
def __init__(
|
||||||
|
self,
|
||||||
|
config_paths: [Path] = None,
|
||||||
|
debug: bool = False,
|
||||||
|
systemd: bool = False,
|
||||||
|
write_to_syslog: bool = False,
|
||||||
|
do_test_logs: bool = True,
|
||||||
|
):
|
||||||
|
self.__do_test_logs = do_test_logs
|
||||||
|
|
||||||
self.__logger = Logger(name=type(self).__name__, debug=debug)
|
self.__logger = Logger(
|
||||||
self.__config_helper = Config(logger=self.__logger)
|
name=type(self).__name__,
|
||||||
|
debug=debug,
|
||||||
|
systemd=systemd,
|
||||||
|
write_to_syslog=write_to_syslog,
|
||||||
|
do_test_logs=do_test_logs,
|
||||||
|
)
|
||||||
|
|
||||||
self.__dry_run = False
|
self.__config = Config(
|
||||||
self.__configs = []
|
logger=self.__logger,
|
||||||
self.__config_paths = []
|
config_files_paths=config_paths
|
||||||
|
)
|
||||||
|
|
||||||
|
self.__global_dry_run = True
|
||||||
self.__calculated_actions = []
|
self.__calculated_actions = []
|
||||||
|
|
||||||
def run(self, configs, dry_run: bool = False):
|
def run(self, global_dry_run: bool = True):
|
||||||
|
|
||||||
self.info("Begin")
|
self.info("Begin rotating")
|
||||||
|
|
||||||
self.__dry_run = dry_run
|
self.__global_dry_run = global_dry_run
|
||||||
self.__config_paths = configs
|
if self.__global_dry_run:
|
||||||
|
self.info(f"Running as a dry run, globally.")
|
||||||
self._consume_configs(self.__config_paths)
|
else:
|
||||||
|
self.info(f"Won't run as a global dry run.")
|
||||||
|
|
||||||
# Rotate once per config
|
# Rotate once per config
|
||||||
for config_index in range(len(self.__configs)):
|
config_file_index = -1
|
||||||
|
for config_file in self.__config.config_files:
|
||||||
|
|
||||||
#
|
config_file: ConfigFile
|
||||||
config = self.__configs[config_index]
|
config_file_index += 1
|
||||||
|
|
||||||
#
|
self.info(
|
||||||
self.info("Rotating for config " + str(config_index + 1) + " of " + str(len(self.__configs)), config["__path"])
|
f"Rotating for config {config_file_index + 1} of {len(self.__config.config_files)}"
|
||||||
self._do_rotate(config)
|
f" : {config_file.path}"
|
||||||
|
f"\n{config_file}"
|
||||||
|
)
|
||||||
|
self._do_rotate(config_file)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def current_time():
|
def current_time():
|
||||||
@ -66,317 +85,328 @@ class BackupRotator:
|
|||||||
|
|
||||||
def debug(self, s):
|
def debug(self, s):
|
||||||
self.__logger.debug(s)
|
self.__logger.debug(s)
|
||||||
|
|
||||||
def info(self, s):
|
def info(self, s):
|
||||||
self.__logger.info(s)
|
self.__logger.info(s)
|
||||||
|
|
||||||
def warn(self, s):
|
def warn(self, s):
|
||||||
self.__logger.warn(s)
|
self.__logger.warning(s)
|
||||||
|
|
||||||
def error(self, s):
|
def error(self, s):
|
||||||
self.__logger.error(s)
|
self.__logger.error(s)
|
||||||
|
|
||||||
def _consume_configs(self, paths: list=None):
|
def _do_rotate(self, config: ConfigFile):
|
||||||
|
|
||||||
configs = self.__config_helper.gather_valid_configs(paths=paths)
|
self.info(
|
||||||
print("Configs:")
|
f"Rotating for config: {config.path}"
|
||||||
print(configs)
|
)
|
||||||
return
|
if config.dry_run:
|
||||||
for config in configs:
|
self.info(
|
||||||
self._consume_config(path=config)
|
f"Config {config.path.name} is set for a dry run (no deleting)."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.info(
|
||||||
|
f"Config {config.path.name} is not set for a dry run (will delete)."
|
||||||
|
)
|
||||||
|
|
||||||
def _consume_config(self, path: str):
|
self._rotate_paths(config=config)
|
||||||
|
|
||||||
# Open the file
|
|
||||||
f = open(path)
|
|
||||||
if not f:
|
|
||||||
raise Exception("Unable to open config file: " + path)
|
|
||||||
|
|
||||||
# Parse
|
|
||||||
config = yaml.safe_load(f)
|
|
||||||
|
|
||||||
# Add its own path
|
|
||||||
config["__path"] = path
|
|
||||||
|
|
||||||
# Consume to internal
|
|
||||||
self.__configs.append(config)
|
|
||||||
self.info("Consumed config from path:", path)
|
|
||||||
|
|
||||||
def _do_rotate(self, config):
|
def _rotate_paths(self, config: ConfigFile):
|
||||||
|
|
||||||
|
paths = config.rotatable_paths
|
||||||
|
self.info(f"Begin rotating {len(paths)} paths")
|
||||||
|
|
||||||
|
for path in paths:
|
||||||
|
|
||||||
|
path: Path
|
||||||
|
|
||||||
|
self._rotate_path(config=config, path=path)
|
||||||
|
|
||||||
self._rotate_paths(config)
|
def _rotate_path(self, config: ConfigFile, path: Path):
|
||||||
|
|
||||||
|
assert path.is_dir(), (
|
||||||
|
f"Path should be a directory: {path}"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.info(
|
||||||
|
f"Rotating path: {path}"
|
||||||
|
)
|
||||||
|
|
||||||
|
self._rotate_path_for_maximum_items(
|
||||||
|
config=config,
|
||||||
|
path=path,
|
||||||
|
)
|
||||||
|
|
||||||
|
self._rotate_path_for_maximum_age(
|
||||||
|
config=config,
|
||||||
|
path=path,
|
||||||
|
)
|
||||||
|
|
||||||
def _rotate_paths(self, config):
|
def _rotate_path_for_maximum_items(self, config: ConfigFile, path: Path):
|
||||||
|
|
||||||
self.info("Begin rotating " + str(len(config["paths"])) + " paths")
|
assert path.is_dir(), f"Path should be a directory: {path}"
|
||||||
for path in config["paths"]:
|
|
||||||
self._rotate_path(config, path)
|
|
||||||
|
|
||||||
def _rotate_path(self, config, path):
|
|
||||||
|
|
||||||
assert os.path.isdir(path), "Path should be a directory: {}".format(path)
|
if config.maximum_items:
|
||||||
|
self.info(
|
||||||
|
f"Rotating path for a maximum of {config.maximum_items} items: {path}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.info(
|
||||||
|
f"Not configured to rotate for maximum number of items."
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
self.info("Rotating path: {}".format(path))
|
self.info(
|
||||||
|
f"Will gather rotation candidates for maximum number of items."
|
||||||
|
)
|
||||||
|
|
||||||
found_any_rotation_keys = False
|
candidate_items = self._gather_rotation_candidates(config=config, path=path)
|
||||||
if "maximum-items" in config.keys():
|
minimum_items = self._determine_minimum_items(config=config)
|
||||||
found_any_rotation_keys = True
|
|
||||||
self._rotate_path_for_maximum_items(config=config, path=path, max_items=config["maximum-items"])
|
|
||||||
if "maximum-age" in config.keys():
|
|
||||||
found_any_rotation_keys = True
|
|
||||||
self._rotate_path_for_maximum_age(config=config, path=path, max_age_days=config["maximum-age"])
|
|
||||||
|
|
||||||
assert found_any_rotation_keys is True, \
|
|
||||||
"Config needs one of the following keys: \"maximum-items\""
|
|
||||||
|
|
||||||
def _rotate_path_for_maximum_items(self, config, path: str, max_items: int):
|
|
||||||
|
|
||||||
assert os.path.isdir(path), "Path should be a directory: {}".format(path)
|
|
||||||
|
|
||||||
self.info("Rotating path for a maximum of {} items: {}".format(
|
|
||||||
max_items, path
|
|
||||||
))
|
|
||||||
|
|
||||||
children = self._gather_rotation_candidates(config, path)
|
|
||||||
|
|
||||||
minimum_items = self._determine_minimum_items(config)
|
|
||||||
|
|
||||||
# Do we need to rotate anything out?
|
# Do we need to rotate anything out?
|
||||||
if len(children) < minimum_items:
|
if len(candidate_items) < minimum_items:
|
||||||
self.info("Path only has {} items, which does not meet the minimum threshold of {} items. Won't rotate this path.".format(
|
|
||||||
len(children), minimum_items
|
self.info(
|
||||||
))
|
f"Path only has {len(candidate_items)} items"
|
||||||
|
f", which does not meet the minimum threshold of {minimum_items} items."
|
||||||
|
" Won't rotate this path."
|
||||||
|
)
|
||||||
return
|
return
|
||||||
elif len(children) <= max_items:
|
|
||||||
self.info("Path only has {} items, but needs more than {} for rotation; Won't rotate this path.".format(
|
elif len(candidate_items) <= config.maximum_items:
|
||||||
len(children), max_items
|
self.info(
|
||||||
))
|
f"Path only has {len(candidate_items)} items"
|
||||||
|
f", but needs more than {config.maximum_items} for rotation"
|
||||||
|
"; Won't rotate this path."
|
||||||
|
)
|
||||||
return
|
return
|
||||||
self.info("Found {} items to examine".format(len(children)))
|
|
||||||
|
self.info(f"Found {len(candidate_items)} items to examine")
|
||||||
|
|
||||||
#
|
#
|
||||||
maximum_purge_count = len(children) - minimum_items
|
maximum_purge_count = len(candidate_items) - minimum_items
|
||||||
purge_count = len(children) - max_items
|
purge_count = len(candidate_items) - config.maximum_items
|
||||||
self.info("Want to purge {} items".format(purge_count))
|
self.info(
|
||||||
|
f"Want to purge {purge_count} items to stay under maximum of {config.maximum_items}"
|
||||||
|
)
|
||||||
|
|
||||||
if purge_count > maximum_purge_count:
|
if purge_count > maximum_purge_count:
|
||||||
self.info("Reducing purge count from {} to {} items to respect minimum items setting ({})".format(
|
self.info(
|
||||||
purge_count, maximum_purge_count, minimum_items
|
f"Reducing purge count from"
|
||||||
))
|
f" {purge_count} to {maximum_purge_count} items"
|
||||||
|
f" to respect minimum items setting ({minimum_items})"
|
||||||
|
)
|
||||||
purge_count = maximum_purge_count
|
purge_count = maximum_purge_count
|
||||||
|
|
||||||
children_to_purge = []
|
items_to_purge = []
|
||||||
for purge_index in range(purge_count):
|
for purge_index in range(purge_count):
|
||||||
|
|
||||||
#
|
#
|
||||||
item_to_purge, item_ctime, item_age_seconds, item_age = self._pick_oldest_item(config, children)
|
item_to_purge, item_ctime, item_age_seconds, item_age = self._pick_oldest_item(
|
||||||
children.remove(item_to_purge)
|
config=config, items=candidate_items
|
||||||
self.info("Found next item to purge: ({}) {} ({})".format(
|
)
|
||||||
purge_index + 1,
|
item_to_purge: Path
|
||||||
os.path.basename(item_to_purge),
|
|
||||||
item_age
|
candidate_items.remove(item_to_purge)
|
||||||
))
|
|
||||||
|
self.info(
|
||||||
|
f"Will purge: ({purge_index + 1})"
|
||||||
|
f" {item_to_purge.name}"
|
||||||
|
f" ({item_age})"
|
||||||
|
)
|
||||||
|
|
||||||
#
|
#
|
||||||
children_to_purge.append(item_to_purge)
|
items_to_purge.append(item_to_purge)
|
||||||
|
|
||||||
#
|
#
|
||||||
self.info("Removing items")
|
self.info("Removing items")
|
||||||
for child_to_purge in children_to_purge:
|
for item_to_purge in items_to_purge:
|
||||||
child_basename = os.path.basename(child_to_purge)
|
|
||||||
self._remove_item(config, child_to_purge)
|
item_to_purge: Path
|
||||||
|
|
||||||
|
self.debug(f"Purging item: {item_to_purge.name}")
|
||||||
|
|
||||||
|
self._remove_item(config=config, path=item_to_purge)
|
||||||
|
|
||||||
def _rotate_path_for_maximum_age(self, config, path: str, max_age_days: int):
|
def _rotate_path_for_maximum_age(self, config: ConfigFile, path: Path):
|
||||||
|
|
||||||
assert os.path.isdir(path), "Path should be a directory: {}".format(path)
|
assert path.is_dir(), f"Path should be a directory: {path}"
|
||||||
|
|
||||||
self.info("Rotating path for max age of {} days: {}".format(max_age_days, path))
|
if config.maximum_age:
|
||||||
|
self.info(
|
||||||
children = self._gather_rotation_candidates(config, path)
|
f"Rotating path for max age of {config.maximum_age} days: {path}"
|
||||||
minimum_items = self._determine_minimum_items(config)
|
)
|
||||||
|
else:
|
||||||
# Do we need to rotate anything out?
|
self.info(
|
||||||
if len(children) < minimum_items:
|
f"Not configured to rotate for a maximum number of days."
|
||||||
self.info("Path only has {} items, which does not meet the minimum threshold of {} items. Won't rotate this path.".format(
|
)
|
||||||
len(children), minimum_items
|
|
||||||
))
|
|
||||||
return
|
return
|
||||||
|
|
||||||
self.info("Examining {} items for deletion".format(len(children)))
|
self.info(
|
||||||
children_to_delete = []
|
f"Will gather rotation candidates for maximum age, in days."
|
||||||
for child in children:
|
)
|
||||||
|
candidate_items = self._gather_rotation_candidates(config=config, path=path)
|
||||||
|
minimum_items = self._determine_minimum_items(config=config)
|
||||||
|
|
||||||
|
# Do we need to rotate anything out?
|
||||||
|
if len(candidate_items) < minimum_items:
|
||||||
|
self.info(
|
||||||
|
f"Path only has {len(candidate_items)} items"
|
||||||
|
f", which does not meet the minimum threshold of {minimum_items} items."
|
||||||
|
f" Won't rotate this path."
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
self.info(
|
||||||
|
f"Examining {len(candidate_items)} items for deletion"
|
||||||
|
)
|
||||||
|
items_to_delete = []
|
||||||
|
for item in candidate_items:
|
||||||
|
|
||||||
age_seconds = self._detect_item_age_seconds(config, child)
|
age_seconds = Util.detect_item_age_seconds(config=config, item=item)
|
||||||
age_days = self._detect_item_age_days(config, child)
|
age_days = Util.detect_item_age_days(config=config, item=item)
|
||||||
age_formatted = self.seconds_to_time_string(age_seconds)
|
age_formatted = Util.seconds_to_time_string(age_seconds)
|
||||||
child_basename = os.path.basename(child)
|
|
||||||
|
|
||||||
if age_days > max_age_days:
|
if age_days > config.maximum_age:
|
||||||
self.info("[Old enough ] {} ({})".format(
|
self.info(
|
||||||
child_basename, age_formatted
|
f"[Old enough ] {item.name} ({age_formatted})"
|
||||||
))
|
)
|
||||||
children_to_delete.append(child)
|
items_to_delete.append(item)
|
||||||
else:
|
else:
|
||||||
self.info("[Not Old enough] {} ({})".format(
|
self.info(
|
||||||
child_basename, age_formatted
|
f"[Not Old enough] {item.name} ({age_formatted})"
|
||||||
))
|
)
|
||||||
|
|
||||||
if len(children_to_delete) > 0:
|
if len(items_to_delete) > 0:
|
||||||
|
|
||||||
self.info("Removing old items ...")
|
self.info("Removing old items ...")
|
||||||
for child_to_delete in children_to_delete:
|
|
||||||
basename = os.path.basename(child_to_delete)
|
for item in items_to_delete:
|
||||||
self._remove_item(config, child_to_delete)
|
self._remove_item(config, item)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
self.info("No old items to remove")
|
self.info("No old items to remove")
|
||||||
|
|
||||||
|
def _gather_rotation_candidates(self, config: ConfigFile, path: Path) -> [Path]:
|
||||||
|
|
||||||
@staticmethod
|
self.debug(f"Begin gathering rotation candidates for: {path}")
|
||||||
def _gather_rotation_candidates(config, path):
|
|
||||||
|
|
||||||
candidates = []
|
candidates: [Path] = []
|
||||||
|
|
||||||
if "target-type" not in config.keys():
|
for item in path.iterdir():
|
||||||
raise Exception("Please provide the configuration key: target-type")
|
|
||||||
|
|
||||||
for item_name in os.listdir(path):
|
|
||||||
|
|
||||||
item_path = os.path.join(path, item_name)
|
self.debug(f"Found an item: {item.name}")
|
||||||
|
|
||||||
if config["target-type"] == "file":
|
if config.target_type == "file":
|
||||||
if not os.path.isfile(item_path):
|
|
||||||
|
if not item.is_file():
|
||||||
|
self.debug(f"Not a file; Skipping: {item.name}")
|
||||||
continue
|
continue
|
||||||
elif config["target-type"] == "directory":
|
|
||||||
if not os.path.isdir(item_path):
|
elif config.target_type == "directory":
|
||||||
|
|
||||||
|
if not item.is_dir():
|
||||||
|
self.debug(f"Not a directory; Skipping: {item.name}")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
else:
|
else:
|
||||||
raise Exception("Configuration key \"target-type\" must be \"file\" or \"directory\"")
|
raise Exception(
|
||||||
|
f"Unsupported target type: {config.target_type}"
|
||||||
|
)
|
||||||
|
|
||||||
candidates.append(item_path)
|
candidates.append(item)
|
||||||
|
|
||||||
|
self.__logger.info(f"Returning {len(candidates)} potential candidates to remove.")
|
||||||
|
|
||||||
return candidates
|
return candidates
|
||||||
|
|
||||||
def _pick_oldest_item(self, config, items):
|
def _pick_oldest_item(self, config: ConfigFile, items: [Path]) -> (Path, float, float, str):
|
||||||
|
|
||||||
best_item = None
|
best_item = None
|
||||||
best_ctime = None
|
best_ctime = None
|
||||||
for item in items:
|
for item in items:
|
||||||
|
|
||||||
ctime = self._detect_item_date(config, item)
|
ctime = Util.detect_item_creation_date(config, item)
|
||||||
if best_ctime is None or ctime < best_ctime:
|
if best_ctime is None or ctime < best_ctime:
|
||||||
best_ctime = ctime
|
best_ctime = ctime
|
||||||
best_item = item
|
best_item = item
|
||||||
|
|
||||||
age_seconds = self._detect_item_age_seconds(config, best_item)
|
age_seconds = Util.detect_item_age_seconds(config, best_item)
|
||||||
age_string = self.seconds_to_time_string(age_seconds)
|
age_string = Util.seconds_to_time_string(age_seconds)
|
||||||
|
|
||||||
return best_item, best_ctime, age_seconds, age_string
|
return best_item, best_ctime, age_seconds, age_string
|
||||||
|
|
||||||
@staticmethod
|
def _remove_item(self, config: ConfigFile, path: Path):
|
||||||
def _detect_item_date(config, item):
|
|
||||||
|
|
||||||
assert "date-detection" in config.keys(), "Please provide config key: \"date-detection\""
|
if path.is_file():
|
||||||
detection = config["date-detection"]
|
|
||||||
|
self._remove_file(config=config, file_path=path)
|
||||||
if detection == "file":
|
|
||||||
ctime = os.path.getctime(item)
|
elif path.is_dir():
|
||||||
|
|
||||||
|
self._remove_directory(config=config, dir_path=path)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
raise AssertionError("Invalid value for \"date-detection\"; Should be one of {file}: {}".format(
|
raise AssertionError(
|
||||||
detection
|
f"Don't know how to remove this item: {path}"
|
||||||
))
|
)
|
||||||
|
|
||||||
return ctime
|
|
||||||
|
|
||||||
def _detect_item_age_seconds(self, config, item):
|
|
||||||
|
|
||||||
now = time.time()
|
|
||||||
ctime = self._detect_item_date(config, item)
|
|
||||||
delta = now - ctime
|
|
||||||
|
|
||||||
return delta
|
|
||||||
|
|
||||||
def _detect_item_age_days(self, config, item):
|
|
||||||
|
|
||||||
age_seconds = self._detect_item_age_seconds(config, item)
|
|
||||||
age_days = int(age_seconds / 86400)
|
|
||||||
|
|
||||||
return age_days
|
|
||||||
|
|
||||||
def seconds_to_time_string(self, seconds: float):
|
|
||||||
|
|
||||||
if isinstance(seconds, float):
|
|
||||||
pass
|
|
||||||
elif isinstance(seconds, int):
|
|
||||||
seconds = float * 1.0
|
|
||||||
else:
|
|
||||||
raise AssertionError("Seconds must be an int or float")
|
|
||||||
|
|
||||||
# Map
|
|
||||||
map = {
|
|
||||||
"year": 31536000.0,
|
|
||||||
"month": 2592000.0,
|
|
||||||
"week": 604800.0,
|
|
||||||
"day": 86400.0,
|
|
||||||
"hour": 3600.0,
|
|
||||||
"minute": 60.0,
|
|
||||||
"second": 1.0
|
|
||||||
}
|
|
||||||
|
|
||||||
s_parts = []
|
|
||||||
for unit_label in map.keys():
|
|
||||||
unit_seconds = map[unit_label]
|
|
||||||
if seconds >= unit_seconds:
|
|
||||||
unit_count = int(seconds / unit_seconds)
|
|
||||||
s_parts.append("{} {}{}".format(
|
|
||||||
unit_count, unit_label,
|
|
||||||
"" if unit_count == 1 else "s"
|
|
||||||
))
|
|
||||||
seconds -= unit_seconds * unit_count
|
|
||||||
|
|
||||||
s = ", ".join(s_parts)
|
|
||||||
|
|
||||||
return s
|
|
||||||
|
|
||||||
def _remove_item(self, config, path):
|
|
||||||
|
|
||||||
if os.path.isfile(path):
|
|
||||||
self._remove_file(config, path)
|
|
||||||
elif os.path.isdir(path):
|
|
||||||
self._remove_directory(config, path)
|
|
||||||
else:
|
|
||||||
raise AssertionError("Don't know how to remove this item: {}".format(path))
|
|
||||||
|
|
||||||
def _remove_file(self, config, file_path):
|
|
||||||
|
|
||||||
if not os.path.isfile(file_path):
|
|
||||||
raise Exception("Tried to remove a file, but this path isn't a file: " + str(file_path))
|
|
||||||
|
|
||||||
if self.__dry_run:
|
|
||||||
self.info("Won't purge file during global-level dry run: ", file_path)
|
|
||||||
elif "dry-run" in config.keys() and config["dry-run"] is True:
|
|
||||||
self.info("Won't purge file during config-level dry run: ", file_path)
|
|
||||||
else:
|
|
||||||
self.info("Purging file:", file_path)
|
|
||||||
os.remove(file_path)
|
|
||||||
|
|
||||||
def _remove_directory(self, config, dir_path):
|
def _remove_file(self, config: ConfigFile, file_path: Path):
|
||||||
|
|
||||||
if not os.path.isdir(dir_path):
|
if not file_path.is_file():
|
||||||
raise Exception("Tried to remove a directory, but this path isn't a directory: " + str(dir_path))
|
raise Exception(
|
||||||
|
f"Tried to remove a file, but this path isn't a file: {file_path}"
|
||||||
|
)
|
||||||
|
|
||||||
if self.__dry_run:
|
if self.__global_dry_run:
|
||||||
self.info("Won't purge directory during global-level dry run: ", dir_path)
|
|
||||||
elif "dry-run" in config.keys() and config["dry-run"] is True:
|
self.info(f"(Global Dry Run) {file_path}")
|
||||||
self.info("Won't purge directory during config-level dry run: ", dir_path)
|
|
||||||
|
elif config.dry_run is True:
|
||||||
|
|
||||||
|
self.info(f"(Config Dry Run) {file_path}")
|
||||||
|
|
||||||
else:
|
else:
|
||||||
self.info("Purging directory:", dir_path)
|
self.info(f"Purging file: {file_path}")
|
||||||
|
file_path.unlink()
|
||||||
|
|
||||||
|
def _remove_directory(self, config: ConfigFile, dir_path: Path):
|
||||||
|
|
||||||
|
if not dir_path.is_dir():
|
||||||
|
raise Exception(
|
||||||
|
f"Tried to remove a directory"
|
||||||
|
f", but this path isn't a directory: {dir_path}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if self.__global_dry_run:
|
||||||
|
|
||||||
|
self.info(f"(Global Dry Run) {dir_path}")
|
||||||
|
|
||||||
|
elif config.dry_run:
|
||||||
|
|
||||||
|
self.info(f"(Config Dry Run) {dir_path}")
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
self.info(f"Purging directory: {dir_path}")
|
||||||
shutil.rmtree(dir_path)
|
shutil.rmtree(dir_path)
|
||||||
|
|
||||||
|
|
||||||
def _determine_minimum_items(self, config):
|
def _determine_minimum_items(self, config) -> int:
|
||||||
|
|
||||||
minimum_items = 0
|
minimum_items = 0
|
||||||
|
|
||||||
if "minimum-items" in config.keys():
|
if config.minimum_items is not None:
|
||||||
minimum_items = config["minimum-items"]
|
|
||||||
self.info("Won't delete anything unless a minimum of {} items were found".format(minimum_items))
|
minimum_items = config.minimum_items
|
||||||
|
|
||||||
|
self.info(
|
||||||
|
f"Won't delete anything unless a minimum of {minimum_items} items were found"
|
||||||
|
)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
self.info("No value found for \"minimum-items\"; Will not enforce minimum item constraint.")
|
self.info(
|
||||||
|
"No minimum number of items specified"
|
||||||
|
"; Will not enforce minimum item constraint."
|
||||||
|
)
|
||||||
|
|
||||||
return minimum_items
|
return minimum_items
|
||||||
|
@ -4,23 +4,45 @@ from logging.handlers import SysLogHandler
|
|||||||
|
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
|
||||||
class Logger:
|
class Logger:
|
||||||
|
|
||||||
def __init__(self, name: str, debug: bool=False):
|
def __init__(
|
||||||
|
self,
|
||||||
|
name: str,
|
||||||
|
debug: bool = False,
|
||||||
|
write_to_syslog: bool = False,
|
||||||
|
systemd: bool = False,
|
||||||
|
do_test_logs: bool = True,
|
||||||
|
):
|
||||||
|
|
||||||
self.__name = name
|
self.__name = name
|
||||||
|
self.__debug = debug
|
||||||
|
self.__write_to_syslog = write_to_syslog
|
||||||
|
self.__systemd = systemd
|
||||||
|
self.__do_test_logs = do_test_logs
|
||||||
|
|
||||||
|
self._init_logger()
|
||||||
|
|
||||||
|
def _init_logger(self):
|
||||||
|
|
||||||
self.__logger = logging.getLogger(self.__name)
|
self.__logger = logging.getLogger(self.__name)
|
||||||
|
|
||||||
if debug:
|
if self.__debug:
|
||||||
level = logging.DEBUG
|
level = logging.DEBUG
|
||||||
else:
|
else:
|
||||||
level = logging.INFO
|
level = logging.INFO
|
||||||
|
|
||||||
self.__logger.setLevel(level)
|
self.__logger.setLevel(level)
|
||||||
|
|
||||||
formatter = logging.Formatter('[%(name)s][%(levelname)s] %(message)s')
|
formatter = logging.Formatter(
|
||||||
formatter_full = logging.Formatter('[%(asctime)s][%(name)s][%(levelname)s] %(message)s')
|
fmt="[{name}][{levelname:<7}] {message}",
|
||||||
|
style='{'
|
||||||
|
)
|
||||||
|
formatter_full = logging.Formatter(
|
||||||
|
fmt="[{asctime}][{name}][{levelname:<7}] {message}",
|
||||||
|
style='{'
|
||||||
|
)
|
||||||
|
|
||||||
# Console output / stream handler (STDOUT)
|
# Console output / stream handler (STDOUT)
|
||||||
handler = logging.StreamHandler(
|
handler = logging.StreamHandler(
|
||||||
@ -28,7 +50,9 @@ class Logger:
|
|||||||
)
|
)
|
||||||
handler.setLevel(level)
|
handler.setLevel(level)
|
||||||
handler.addFilter(lambda entry: entry.levelno <= logging.INFO)
|
handler.addFilter(lambda entry: entry.levelno <= logging.INFO)
|
||||||
handler.setFormatter(formatter_full)
|
handler.setFormatter(
|
||||||
|
formatter if self.__systemd else formatter_full
|
||||||
|
)
|
||||||
self.__logger.addHandler(handler)
|
self.__logger.addHandler(handler)
|
||||||
|
|
||||||
# Console output / stream handler (STDERR)
|
# Console output / stream handler (STDERR)
|
||||||
@ -36,28 +60,38 @@ class Logger:
|
|||||||
stream=sys.stderr
|
stream=sys.stderr
|
||||||
)
|
)
|
||||||
handler.setLevel(logging.WARNING)
|
handler.setLevel(logging.WARNING)
|
||||||
handler.setFormatter(formatter_full)
|
handler.setFormatter(
|
||||||
|
formatter if self.__systemd else formatter_full
|
||||||
|
)
|
||||||
self.__logger.addHandler(handler)
|
self.__logger.addHandler(handler)
|
||||||
|
|
||||||
# Syslog handler
|
# Syslog handler
|
||||||
handler = SysLogHandler(
|
if self.__write_to_syslog:
|
||||||
address="/dev/log"
|
handler = SysLogHandler(
|
||||||
)
|
address="/dev/log"
|
||||||
handler.setLevel(level)
|
)
|
||||||
handler.setFormatter(formatter)
|
handler.setLevel(level)
|
||||||
self.__logger.addHandler(handler)
|
handler.setFormatter(formatter)
|
||||||
|
self.__logger.addHandler(handler)
|
||||||
self.debug("Test debug log")
|
|
||||||
self.info("Test info log")
|
|
||||||
self.warn("Test warn log")
|
|
||||||
self.error("Test error log")
|
|
||||||
|
|
||||||
|
# This is annoying inside cron
|
||||||
|
if self.__do_test_logs:
|
||||||
|
self.debug("Test debug log")
|
||||||
|
self.info("Test info log")
|
||||||
|
self.warn("Test warn log")
|
||||||
|
self.error("Test error log")
|
||||||
|
|
||||||
def debug(self, s):
|
def debug(self, s):
|
||||||
self.__logger.debug(s)
|
self.__logger.debug(s)
|
||||||
|
|
||||||
def info(self, s):
|
def info(self, s):
|
||||||
self.__logger.info(s)
|
self.__logger.info(s)
|
||||||
|
|
||||||
def warn(self, s):
|
def warn(self, s):
|
||||||
self.__logger.warn(s)
|
self.__logger.warning(s)
|
||||||
|
|
||||||
|
def warning(self, s):
|
||||||
|
self.__logger.warning(s)
|
||||||
|
|
||||||
def error(self, s):
|
def error(self, s):
|
||||||
self.__logger.error(s)
|
self.__logger.error(s)
|
||||||
|
128
domain/Util.py
Normal file
128
domain/Util.py
Normal file
@ -0,0 +1,128 @@
|
|||||||
|
|
||||||
|
|
||||||
|
from domain.config.ConfigFile import ConfigFile
|
||||||
|
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
class Util:
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_dir_files_recursive(path: Path) -> [Path]:
|
||||||
|
|
||||||
|
files_paths = []
|
||||||
|
for dir_path, dirs_names, filenames in path.walk():
|
||||||
|
|
||||||
|
for file_name in filenames:
|
||||||
|
|
||||||
|
file_path = dir_path / file_name
|
||||||
|
|
||||||
|
files_paths.append(file_path)
|
||||||
|
|
||||||
|
return files_paths
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def detect_item_creation_date(config: ConfigFile, item: Path) -> datetime.datetime:
|
||||||
|
|
||||||
|
stat = None
|
||||||
|
|
||||||
|
if config.date_detection == "file":
|
||||||
|
|
||||||
|
# Try for the most accurate stat
|
||||||
|
# First one that raises will just break the block, obv
|
||||||
|
try:
|
||||||
|
stat = item.stat().st_ctime
|
||||||
|
# print("got ctime")
|
||||||
|
stat = item.stat().st_mtime
|
||||||
|
# print("got mtime")
|
||||||
|
stat = item.stat().st_birthtime
|
||||||
|
# print("got btime")
|
||||||
|
except AttributeError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise AssertionError(
|
||||||
|
f"Unsupported date-detection option: {config.date_detection}"
|
||||||
|
)
|
||||||
|
|
||||||
|
stamp = datetime.datetime.fromtimestamp(
|
||||||
|
stat
|
||||||
|
)
|
||||||
|
# print("Stat:", stat)
|
||||||
|
# print("Stamp:", stamp)
|
||||||
|
# print(item.name, "==>", stamp)
|
||||||
|
|
||||||
|
return stamp
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def detect_item_age_seconds(config: ConfigFile, item: Path) -> float:
|
||||||
|
|
||||||
|
now = datetime.datetime.now()
|
||||||
|
|
||||||
|
ctime = Util.detect_item_creation_date(config=config, item=item)
|
||||||
|
delta = now - ctime
|
||||||
|
seconds = delta.seconds
|
||||||
|
|
||||||
|
# print(item.name, "==>", seconds, f"({ctime})")
|
||||||
|
# print(">", "Now was:", now)
|
||||||
|
# print(">", "ctime was:", ctime)
|
||||||
|
# print(">", "Delta was:", delta)
|
||||||
|
# print(">", "Seconds was:", delta.total_seconds())
|
||||||
|
|
||||||
|
return delta.total_seconds()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def detect_item_age_days(config: ConfigFile, item: Path) -> int:
|
||||||
|
|
||||||
|
age_seconds = Util.detect_item_age_seconds(
|
||||||
|
config=config, item=item
|
||||||
|
)
|
||||||
|
age_days = int(age_seconds / 86400)
|
||||||
|
|
||||||
|
return age_days
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def seconds_to_time_string(seconds: float):
|
||||||
|
|
||||||
|
if isinstance(seconds, float):
|
||||||
|
pass
|
||||||
|
elif isinstance(seconds, int):
|
||||||
|
seconds = float(seconds)
|
||||||
|
else:
|
||||||
|
raise AssertionError("Seconds must be an int or float")
|
||||||
|
|
||||||
|
# Map
|
||||||
|
dt_map = {
|
||||||
|
"year": 31536000.0,
|
||||||
|
"month": 2592000.0,
|
||||||
|
"week": 604800.0,
|
||||||
|
"day": 86400.0,
|
||||||
|
"hour": 3600.0,
|
||||||
|
"minute": 60.0,
|
||||||
|
"second": 1.0
|
||||||
|
}
|
||||||
|
|
||||||
|
s_parts = []
|
||||||
|
for unit_label in dt_map.keys():
|
||||||
|
|
||||||
|
unit_seconds = dt_map[unit_label]
|
||||||
|
|
||||||
|
if seconds >= unit_seconds:
|
||||||
|
|
||||||
|
unit_count = int(seconds / unit_seconds)
|
||||||
|
|
||||||
|
unit_plural = "" if unit_count == 1 else "s"
|
||||||
|
s_parts.append(
|
||||||
|
f"{unit_count} {unit_label}{unit_plural}"
|
||||||
|
)
|
||||||
|
|
||||||
|
seconds -= unit_seconds * unit_count
|
||||||
|
|
||||||
|
s = ", ".join(s_parts)
|
||||||
|
|
||||||
|
return s
|
43
domain/config/Config.py
Normal file
43
domain/config/Config.py
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
|
||||||
|
|
||||||
|
from domain.config.ConfigFile import ConfigFile
|
||||||
|
from domain.config.Scanner import Scanner
|
||||||
|
from domain.Logger import Logger
|
||||||
|
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
|
||||||
|
def __init__(self, logger: Logger, config_files_paths: [Path]):
|
||||||
|
|
||||||
|
self.__logger = logger
|
||||||
|
|
||||||
|
self.__config_files_paths: [Path] = config_files_paths
|
||||||
|
self.__configs = {}
|
||||||
|
|
||||||
|
self.__scanner = Scanner(
|
||||||
|
logger=self.__logger
|
||||||
|
)
|
||||||
|
|
||||||
|
self._consume_configs()
|
||||||
|
|
||||||
|
def _consume_configs(self):
|
||||||
|
|
||||||
|
config_paths = self.__scanner.gather_valid_config_paths(
|
||||||
|
paths=self.__config_files_paths
|
||||||
|
)
|
||||||
|
|
||||||
|
for config_path in config_paths:
|
||||||
|
|
||||||
|
config = ConfigFile(
|
||||||
|
logger=self.__logger,
|
||||||
|
path=config_path
|
||||||
|
)
|
||||||
|
|
||||||
|
self.__configs[config.key] = config
|
||||||
|
|
||||||
|
@property
|
||||||
|
def config_files(self) -> [ConfigFile]:
|
||||||
|
return self.__configs.values()
|
309
domain/config/ConfigFile.py
Normal file
309
domain/config/ConfigFile.py
Normal file
@ -0,0 +1,309 @@
|
|||||||
|
|
||||||
|
|
||||||
|
from domain.Logger import Logger
|
||||||
|
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
|
||||||
|
class ConfigFile:
|
||||||
|
|
||||||
|
__VALID_TARGET_TYPES = [
|
||||||
|
"file",
|
||||||
|
"directory"
|
||||||
|
]
|
||||||
|
|
||||||
|
__VALID_DATE_DETECTION_TYPES = [
|
||||||
|
"file"
|
||||||
|
]
|
||||||
|
|
||||||
|
__DEFAULT_MINIMUM_ITEMS = 0
|
||||||
|
__DEFAULT_MAXIMUM_ITEMS = None
|
||||||
|
__DEFAULT_MAXIMUM_AGE = None
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self, logger: Logger,
|
||||||
|
path: Path,
|
||||||
|
):
|
||||||
|
|
||||||
|
self.__logger = logger
|
||||||
|
self.__path = path.absolute()
|
||||||
|
|
||||||
|
# noinspection PyTypeChecker
|
||||||
|
self.__data: dict = None
|
||||||
|
|
||||||
|
self.__dry_run: bool = True
|
||||||
|
|
||||||
|
# noinspection PyTypeChecker
|
||||||
|
self.__target_type: str = None
|
||||||
|
|
||||||
|
# noinspection PyTypeChecker
|
||||||
|
self.__date_detection: str = None
|
||||||
|
|
||||||
|
self.__rotatable_paths: [Path] = []
|
||||||
|
|
||||||
|
self.__minimum_items = self.__DEFAULT_MINIMUM_ITEMS
|
||||||
|
# noinspection PyTypeChecker
|
||||||
|
self.__maximum_items: int = self.__DEFAULT_MAXIMUM_ITEMS
|
||||||
|
# noinspection PyTypeChecker
|
||||||
|
self.__maximum_age: int = None
|
||||||
|
|
||||||
|
self._load()
|
||||||
|
self._consume()
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
|
||||||
|
s = ""
|
||||||
|
|
||||||
|
s += "*** Config File ***"
|
||||||
|
s += f"\n> Path: {self.__path}"
|
||||||
|
s += f"\n> Dry run: " + ("Yes" if self.__dry_run else "No")
|
||||||
|
s += f"\n> Minimum items: {self.__minimum_items}"
|
||||||
|
s += f"\n> Maximum items: {self.__maximum_items}"
|
||||||
|
s += f"\n> Maximum age (in days): {self.__maximum_age}"
|
||||||
|
s += f"\n> Target type: {self.__target_type}"
|
||||||
|
s += f"\n> Date detection: {self.__date_detection}"
|
||||||
|
s += f"\n> Rotatable paths: "
|
||||||
|
if len(self.__rotatable_paths) > 0:
|
||||||
|
for p in self.__rotatable_paths:
|
||||||
|
s += f"\n>> {p}"
|
||||||
|
else:
|
||||||
|
s += "\n>> [none]"
|
||||||
|
|
||||||
|
return s
|
||||||
|
|
||||||
|
def _load(self):
|
||||||
|
|
||||||
|
self.info(f"Loading config: {self.__path}")
|
||||||
|
|
||||||
|
assert self.__path.is_file(), (
|
||||||
|
f"Cannot load config file because it isn't a file: {self.__path}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Open the file
|
||||||
|
self.debug(f"Opening config file for load: {self.__path}")
|
||||||
|
f = open(str(self.__path))
|
||||||
|
if not f:
|
||||||
|
raise Exception(f"Unable to open config file: {self.__path}")
|
||||||
|
|
||||||
|
# Load data
|
||||||
|
self.__data = yaml.safe_load(f)
|
||||||
|
assert self.__data is not None, (
|
||||||
|
f"Config file seems to be null or empty: {self.__path}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Consume to internal
|
||||||
|
self.info(f"Loaded config from path: {self.__path}")
|
||||||
|
|
||||||
|
def _consume(self):
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
assert isinstance(self.__data, dict), (
|
||||||
|
f"Config file should be a dict!"
|
||||||
|
)
|
||||||
|
|
||||||
|
if "options" in self.__data.keys():
|
||||||
|
|
||||||
|
self.info(f"Found options setting")
|
||||||
|
options = self.__data["options"]
|
||||||
|
assert isinstance(options, dict), "Options must be a dict"
|
||||||
|
|
||||||
|
if "dry-run" in options.keys():
|
||||||
|
|
||||||
|
dry_run = self.__data["options"]["dry-run"]
|
||||||
|
self.info(f"Found dry run option: {dry_run}")
|
||||||
|
assert isinstance(dry_run, bool), "dry-run setting must be boolean"
|
||||||
|
self.__dry_run = dry_run
|
||||||
|
else:
|
||||||
|
self.warning(f"No dry-run option found; Will use default: {self.__dry_run}")
|
||||||
|
|
||||||
|
if "minimum-items" in options.keys():
|
||||||
|
|
||||||
|
minimum_items = options["minimum-items"]
|
||||||
|
self.info(f"Found minimum-items option: {minimum_items}")
|
||||||
|
if minimum_items is None:
|
||||||
|
minimum_items = self.__DEFAULT_MINIMUM_ITEMS
|
||||||
|
assert isinstance(minimum_items, int), (
|
||||||
|
f"Option minimum-items must be an integer,"
|
||||||
|
f" but got: {type(minimum_items).__name__} ({minimum_items})"
|
||||||
|
)
|
||||||
|
self.__minimum_items = minimum_items
|
||||||
|
else:
|
||||||
|
self.warning(
|
||||||
|
f"No minimum-items option found; Will use default: {self.__minimum_items}"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (
|
||||||
|
"maximum-items" in options.keys()
|
||||||
|
or
|
||||||
|
"maximum-age" in options.keys()
|
||||||
|
), (
|
||||||
|
"Options should include either maximum-items or maximum-age"
|
||||||
|
)
|
||||||
|
|
||||||
|
if "maximum-items" in options.keys():
|
||||||
|
|
||||||
|
maximum_items = options["maximum-items"]
|
||||||
|
self.info(f"Found maximum-items option: {maximum_items}")
|
||||||
|
assert maximum_items is None or isinstance(maximum_items, int), (
|
||||||
|
f"Option maximum-items must be integer, but got: {maximum_items}"
|
||||||
|
)
|
||||||
|
assert maximum_items is None or maximum_items > 0, (
|
||||||
|
f"Option maximum-items is zero, which doesn't make sense."
|
||||||
|
)
|
||||||
|
self.__maximum_items = maximum_items
|
||||||
|
else:
|
||||||
|
self.warning(
|
||||||
|
f"No maximum-items option found; Will use default: {self.__maximum_items}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if "maximum-age" in options.keys():
|
||||||
|
|
||||||
|
maximum_age = options["maximum-age"]
|
||||||
|
self.info(f"Found maximum-age option (max age in days): {maximum_age}")
|
||||||
|
assert maximum_age is None or isinstance(maximum_age, int), (
|
||||||
|
f"Option maximum-age must be None or an integer,"
|
||||||
|
f" but got: {type(maximum_age).__name__} ({maximum_age})"
|
||||||
|
)
|
||||||
|
assert maximum_age is None or maximum_age > 0, (
|
||||||
|
f"Option maximum-age is zero, which doesn't make sense."
|
||||||
|
)
|
||||||
|
self.__maximum_age = maximum_age
|
||||||
|
else:
|
||||||
|
self.warning(
|
||||||
|
f"No maximum-age option found; Will use default: {self.__maximum_age}"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "target-type" in options.keys(), (
|
||||||
|
f"Option target-type is required"
|
||||||
|
)
|
||||||
|
target_type = options["target-type"]
|
||||||
|
self.info(f"Found target-type option: {target_type}")
|
||||||
|
assert isinstance(target_type, str), (
|
||||||
|
f"Option target-type must be str, but got: {target_type}"
|
||||||
|
)
|
||||||
|
assert target_type in self.__VALID_TARGET_TYPES, (
|
||||||
|
f"Option target-type must be one of: {self.__VALID_TARGET_TYPES}"
|
||||||
|
)
|
||||||
|
self.__target_type = target_type
|
||||||
|
|
||||||
|
if "date-detection" in options.keys():
|
||||||
|
date_detection = options["date-detection"]
|
||||||
|
self.info(f"Found date-detection option: {date_detection}")
|
||||||
|
assert isinstance(date_detection, str), (
|
||||||
|
f"Option date-detection must be str, but got: {date_detection}"
|
||||||
|
)
|
||||||
|
assert date_detection in self.__VALID_DATE_DETECTION_TYPES, (
|
||||||
|
f"Option date-detection must be one of: {self.__VALID_DATE_DETECTION_TYPES}"
|
||||||
|
)
|
||||||
|
self.__date_detection = date_detection
|
||||||
|
else:
|
||||||
|
self.error(
|
||||||
|
f"Option date-detection not found; Will use default: {self.__date_detection}"
|
||||||
|
)
|
||||||
|
raise AssertionError(
|
||||||
|
f"Option date-detection is required."
|
||||||
|
)
|
||||||
|
|
||||||
|
else:
|
||||||
|
self.error(f"No options key found!")
|
||||||
|
raise AssertionError(f"No options key found!")
|
||||||
|
|
||||||
|
assert "paths" in self.__data, (
|
||||||
|
f"Could not find 'paths' key"
|
||||||
|
)
|
||||||
|
rotatable_paths = self.__data["paths"]
|
||||||
|
if isinstance(rotatable_paths, str):
|
||||||
|
rotatable_paths = [rotatable_paths]
|
||||||
|
assert isinstance(rotatable_paths, list), (
|
||||||
|
"Rotatable 'paths' key must be a string or list"
|
||||||
|
)
|
||||||
|
for i in range(len(rotatable_paths)):
|
||||||
|
p = rotatable_paths[i]
|
||||||
|
if isinstance(p, Path):
|
||||||
|
continue
|
||||||
|
elif isinstance(p, str):
|
||||||
|
rotatable_paths[i] = Path(p)
|
||||||
|
else:
|
||||||
|
raise AssertionError(
|
||||||
|
f"All rotatable paths must be strings or pathlib::Path objects"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.__rotatable_paths = rotatable_paths
|
||||||
|
self.info(f"Found {len(self.__rotatable_paths)} rotatable paths")
|
||||||
|
|
||||||
|
except KeyError as e:
|
||||||
|
|
||||||
|
self.error(
|
||||||
|
f"Failed to load config due to KeyError"
|
||||||
|
f"\nFile: {self.__path}"
|
||||||
|
f"\nError: {str(e)}"
|
||||||
|
)
|
||||||
|
raise e
|
||||||
|
|
||||||
|
except AssertionError as e:
|
||||||
|
|
||||||
|
self.error(
|
||||||
|
f"Failed to load config due to AssertionError"
|
||||||
|
f"\nFile: {self.__path}"
|
||||||
|
f"\nError: {str(e)}"
|
||||||
|
)
|
||||||
|
raise e
|
||||||
|
|
||||||
|
def debug(self, s):
|
||||||
|
self.__logger.debug(f"({self.__path.name}) {s}")
|
||||||
|
|
||||||
|
def info(self, s):
|
||||||
|
self.__logger.info(f"({self.__path.name}) {s}")
|
||||||
|
|
||||||
|
def warning(self, s):
|
||||||
|
self.__logger.warning(f"({self.__path.name}) {s}")
|
||||||
|
|
||||||
|
def error(self, s):
|
||||||
|
self.__logger.error(f"({self.__path.name}) {s}")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def key(self) -> str:
|
||||||
|
return str(self.__path)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def path(self) -> Path:
|
||||||
|
return self.__path
|
||||||
|
|
||||||
|
@property
|
||||||
|
def data(self) -> dict:
|
||||||
|
return self.__data
|
||||||
|
|
||||||
|
@property
|
||||||
|
def dry_run(self) -> bool:
|
||||||
|
return self.__dry_run
|
||||||
|
|
||||||
|
@dry_run.setter
|
||||||
|
def dry_run(self, b: bool):
|
||||||
|
self.__dry_run = b
|
||||||
|
|
||||||
|
@property
|
||||||
|
def target_type(self) -> str:
|
||||||
|
return self.__target_type
|
||||||
|
|
||||||
|
@property
|
||||||
|
def date_detection(self) -> str:
|
||||||
|
return self.__date_detection
|
||||||
|
|
||||||
|
@property
|
||||||
|
def rotatable_paths(self) -> [Path]:
|
||||||
|
return self.__rotatable_paths
|
||||||
|
|
||||||
|
@property
|
||||||
|
def minimum_items(self) -> int:
|
||||||
|
return self.__minimum_items
|
||||||
|
|
||||||
|
@property
|
||||||
|
def maximum_items(self) -> int:
|
||||||
|
return self.__maximum_items
|
||||||
|
|
||||||
|
@property
|
||||||
|
def maximum_age(self) -> int:
|
||||||
|
return self.__maximum_age
|
@ -1,48 +1,38 @@
|
|||||||
|
|
||||||
|
|
||||||
from domain.Logger import Logger
|
from domain.Logger import Logger
|
||||||
|
from domain.Util import Util
|
||||||
import os
|
|
||||||
|
|
||||||
|
|
||||||
class Config:
|
# import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
class Scanner:
|
||||||
|
|
||||||
__DEFAULT_VALID_EXTENSIONS = [
|
__DEFAULT_VALID_EXTENSIONS = [
|
||||||
"yaml",
|
"yaml",
|
||||||
"yml"
|
"yml"
|
||||||
]
|
]
|
||||||
|
|
||||||
def __init__(self, logger):
|
def __init__(self, logger: Logger):
|
||||||
|
|
||||||
self.__logger = logger
|
self.__logger = logger
|
||||||
self.__valid_extensions = self.__DEFAULT_VALID_EXTENSIONS
|
self.__valid_extensions = self.__DEFAULT_VALID_EXTENSIONS
|
||||||
|
|
||||||
def debug(self, s):
|
def debug(self, s):
|
||||||
self.__logger.debug(f"[{type(self).__name__}] {s}")
|
self.__logger.debug(f"[{type(self).__name__}] {s}")
|
||||||
|
|
||||||
def info(self, s):
|
def info(self, s):
|
||||||
self.__logger.info(f"[{type(self).__name__}] {s}")
|
self.__logger.info(f"[{type(self).__name__}] {s}")
|
||||||
|
|
||||||
def warn(self, s):
|
def warn(self, s):
|
||||||
self.__logger.warn(f"[{type(self).__name__}] {s}")
|
self.__logger.warning(f"[{type(self).__name__}] {s}")
|
||||||
|
|
||||||
def error(self, s):
|
def error(self, s):
|
||||||
self.__logger.error(f"[{type(self).__name__}] {s}")
|
self.__logger.error(f"[{type(self).__name__}] {s}")
|
||||||
|
|
||||||
@staticmethod
|
def gather_valid_config_paths(self, paths: list = None) -> [Path]:
|
||||||
def get_dir_files_recursive(path: str):
|
|
||||||
|
|
||||||
files_paths = []
|
|
||||||
|
|
||||||
for dir_path, dirnames, filenames in os.walk(path):
|
|
||||||
|
|
||||||
for file_name in filenames:
|
|
||||||
|
|
||||||
file_path = os.path.join(dir_path, file_name)
|
|
||||||
files_paths.append(file_path)
|
|
||||||
# print("Uhm yeah", dir_path, "--", dirnames, "--", file_name)
|
|
||||||
# print("==>", file_path)
|
|
||||||
|
|
||||||
return files_paths
|
|
||||||
|
|
||||||
def gather_valid_configs(self, paths: list=None):
|
|
||||||
|
|
||||||
assert paths is not None, "Config paths cannot be None"
|
assert paths is not None, "Config paths cannot be None"
|
||||||
assert len(paths) > 0, "Must provide at least one config file path"
|
assert len(paths) > 0, "Must provide at least one config file path"
|
||||||
@ -50,64 +40,81 @@ class Config:
|
|||||||
self.info("Gathering valid configs")
|
self.info("Gathering valid configs")
|
||||||
|
|
||||||
file_paths = []
|
file_paths = []
|
||||||
configs = []
|
config_paths = []
|
||||||
not_configs = []
|
not_config_paths = []
|
||||||
|
|
||||||
# First gather all files that are potential configs
|
# First gather all files that are potential configs
|
||||||
for path in paths:
|
for path_str in paths:
|
||||||
|
|
||||||
|
path = Path(path_str)
|
||||||
|
|
||||||
self.info(f"Inspecting path: {path}")
|
self.info(f"Inspecting path: {path}")
|
||||||
|
|
||||||
if os.path.isfile(path):
|
if not path.exists():
|
||||||
self.debug(f"Path is a file; Adding directly to potential config candidates: {path}")
|
|
||||||
|
self.error(f"Path doesn't exist: {path}")
|
||||||
|
|
||||||
|
if path.is_file():
|
||||||
|
|
||||||
|
self.debug(
|
||||||
|
f"Path is a file; Adding directly to potential config candidates: {path}"
|
||||||
|
)
|
||||||
file_paths.append(path)
|
file_paths.append(path)
|
||||||
|
|
||||||
elif os.path.isdir(path):
|
elif path.is_dir():
|
||||||
self.debug(f"Path is a dir; Scanning recursively for potential config candidate files: {path}")
|
|
||||||
for file_path in Config.get_dir_files_recursive(path=path):
|
self.debug(
|
||||||
|
f"Path is a dir;"
|
||||||
|
" Scanning recursively for potential config candidate files: {path}"
|
||||||
|
)
|
||||||
|
|
||||||
|
for file_path in Util.get_dir_files_recursive(path=path):
|
||||||
self.info(f"> Candidate file: {file_path}")
|
self.info(f"> Candidate file: {file_path}")
|
||||||
file_paths.append(file_path)
|
file_paths.append(file_path)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
raise AssertionError(f"Don't know how to handle path that isn't a file or dir: {path}")
|
raise AssertionError(
|
||||||
|
f"Don't know how to handle path that isn't a file or dir: {path}"
|
||||||
|
)
|
||||||
|
|
||||||
# Now, filter for files with valid YAML extensions
|
# Now, filter for files with valid YAML extensions
|
||||||
for file_path in file_paths:
|
for file_path in file_paths:
|
||||||
|
|
||||||
if self.check_file_extension(file_path=file_path, extensions=None):
|
if self.check_file_extension(file_path=file_path, extensions=None):
|
||||||
configs.append(file_path)
|
config_paths.append(file_path)
|
||||||
else:
|
else:
|
||||||
not_configs.append(file_path)
|
not_config_paths.append(file_path)
|
||||||
|
|
||||||
self.info("Filtered out non-config files:")
|
self.info("Filtered out non-config files:")
|
||||||
if len(not_configs) > 0:
|
if len(not_config_paths) > 0:
|
||||||
for not_config in not_configs:
|
for not_config_path in not_config_paths:
|
||||||
self.info(f"> {not_config}")
|
self.info(f"> {not_config_path}")
|
||||||
else:
|
else:
|
||||||
self.info("> [none]")
|
self.info("> [none]")
|
||||||
|
|
||||||
self.info("Kept config-looking files:")
|
self.info("Kept config-looking files:")
|
||||||
if len(configs) > 0:
|
if len(config_paths) > 0:
|
||||||
for config in configs:
|
for config_path in config_paths:
|
||||||
self.info(f"> {config}")
|
self.info(f"> {config_path}")
|
||||||
else:
|
else:
|
||||||
self.info("> [none]")
|
self.info("> [none]")
|
||||||
|
|
||||||
return configs
|
return config_paths
|
||||||
|
|
||||||
def check_file_extension(self, file_path, extensions: list=None):
|
def check_file_extension(self, file_path: Path, extensions: list = None) -> bool:
|
||||||
|
|
||||||
if extensions is None:
|
if extensions is None:
|
||||||
extensions = self.__valid_extensions
|
extensions = self.__valid_extensions
|
||||||
|
|
||||||
file_name, file_extension = os.path.splitext(file_path)
|
file_extension = file_path.suffix
|
||||||
|
|
||||||
|
# Strip preceding dot from extension
|
||||||
if len(file_extension) > 0 and file_extension[0] == ".":
|
if len(file_extension) > 0 and file_extension[0] == ".":
|
||||||
file_extension = file_extension[1:]
|
file_extension = file_extension[1:]
|
||||||
file_extension = file_extension.lower()
|
file_extension = file_extension.lower()
|
||||||
|
|
||||||
for valid_extension in extensions:
|
for valid_extension in extensions:
|
||||||
#print(file_name, "---", file_extension, "---", valid_extension)
|
|
||||||
if file_extension == valid_extension:
|
if file_extension == valid_extension:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
47
init-environment
Executable file
47
init-environment
Executable file
@ -0,0 +1,47 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
log()
|
||||||
|
{
|
||||||
|
echo "[Mike's Backup Rotator - Init Env] $1"
|
||||||
|
}
|
||||||
|
complain()
|
||||||
|
{
|
||||||
|
echo "[Mike's Backup Rotator - Init Env] $1" 1>&2
|
||||||
|
}
|
||||||
|
die()
|
||||||
|
{
|
||||||
|
complain "Fatal: $1"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
SCRIPT_PATH=$(readlink -f "$0")
|
||||||
|
SCRIPT_DIR=$(dirname "$SCRIPT_PATH")
|
||||||
|
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
|
||||||
|
|
||||||
|
log "Begin ${SCRIPT_NAME}"
|
||||||
|
log "Script path: ${SCRIPT_PATH}"
|
||||||
|
log "Script dir: ${SCRIPT_DIR}"
|
||||||
|
log "PATH: ${PATH}"
|
||||||
|
|
||||||
|
log "PWD before switching: $(pwd)"
|
||||||
|
cd "${SCRIPT_DIR}" || die "Failed to switch to project directory: ${SCRIPT_DIR}"
|
||||||
|
log "PWD after switching: $(pwd)"
|
||||||
|
|
||||||
|
log "Printing environment:"
|
||||||
|
printenv
|
||||||
|
|
||||||
|
log "Ensuring python installation with pyenv"
|
||||||
|
pyenv versions
|
||||||
|
pyenv install --skip-existing || die "Failed to ensure python installation with pyenv"
|
||||||
|
|
||||||
|
log "Installing/upgrading pip and pipenv"
|
||||||
|
pip install --upgrade pip pipenv || die "Failed to install/upgrade pip and pipenv"
|
||||||
|
|
||||||
|
log "Removing old pip environment"
|
||||||
|
pipenv --rm # Don't die because this will return an error if the env didn't already exist
|
||||||
|
|
||||||
|
# Install/sync
|
||||||
|
log "Syncing pip dependencies"
|
||||||
|
pipenv sync || die "Failed to sync pip environment with pipenv"
|
||||||
|
|
||||||
|
|
60
main.py
60
main.py
@ -21,30 +21,76 @@ def main():
|
|||||||
help="Verbose/Debug logging mode"
|
help="Verbose/Debug logging mode"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--systemd",
|
||||||
|
default=False,
|
||||||
|
dest="systemd",
|
||||||
|
action="store_true",
|
||||||
|
help=(
|
||||||
|
"Pass if this program will be spawned inside systemd"
|
||||||
|
" or another system that already adds timestamps to log messages."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--syslog", "--write-to-syslog",
|
||||||
|
default=False,
|
||||||
|
dest="write_to_syslog",
|
||||||
|
action="store_true",
|
||||||
|
help=(
|
||||||
|
"Pass if you'd like this program to write to syslog."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--no-test-logs",
|
||||||
|
default=False,
|
||||||
|
dest="do_test_logs",
|
||||||
|
action="store_false",
|
||||||
|
help=(
|
||||||
|
"Pass if you do not want to see test logs for all log levels."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--test-logs",
|
||||||
|
default=True,
|
||||||
|
dest="do_test_logs",
|
||||||
|
action="store_true",
|
||||||
|
help=(
|
||||||
|
"Pass if you want to see test logs for all log levels."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--config", "-c",
|
"--config", "-c",
|
||||||
dest="config_files",
|
dest="config_paths",
|
||||||
default=[],
|
default=[],
|
||||||
action="append",
|
action="append",
|
||||||
type=str,
|
type=str,
|
||||||
help="Specify a configuration file. Can be called multiple times."
|
help="Specify a configuration file or configuration directory. Can be called multiple times."
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--dry-run", "-d",
|
"--dry-run", "-d",
|
||||||
dest="dry_run",
|
dest="global_dry_run",
|
||||||
default=False,
|
default=False,
|
||||||
action="store_true",
|
action="store_true",
|
||||||
help="Only perform an analysis; Don't delete anything."
|
help=(
|
||||||
|
"Only perform an analysis;"
|
||||||
|
" Don't delete anything no matter what configs say (configs can specify dry run, too)."
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
rotator = BackupRotator(
|
rotator = BackupRotator(
|
||||||
debug=args.debug
|
config_paths=args.config_paths,
|
||||||
|
debug=args.debug,
|
||||||
|
systemd=args.systemd,
|
||||||
|
write_to_syslog=args.write_to_syslog,
|
||||||
|
do_test_logs=args.do_test_logs,
|
||||||
)
|
)
|
||||||
rotator.run(
|
rotator.run(
|
||||||
configs=args.config_files,
|
global_dry_run=args.global_dry_run
|
||||||
dry_run=args.dry_run
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
Reference in New Issue
Block a user