543 lines
12 KiB
Python
Executable File
543 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
|
|
#
|
|
import datetime
|
|
import os
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
#
|
|
class MikesBackup:
|
|
|
|
#
|
|
__log_dir = None
|
|
|
|
__remote_host = None
|
|
__remote_user = None
|
|
__destination_dir_base = None
|
|
|
|
__source_dirs = []
|
|
__source_dir_excludes = []
|
|
__source_mountpoint_demands = []
|
|
|
|
__ssh_key = None
|
|
__quiet_ssh = True
|
|
|
|
__force_full = False
|
|
__force_differential = False
|
|
|
|
#
|
|
def __init__(self):
|
|
|
|
self.parse_args()
|
|
|
|
#
|
|
def log(self, s, o=None):
|
|
|
|
the_date = self.get_datetime_for_logging()
|
|
|
|
to_log = "[MikesBackup][" + the_date + "] " + s
|
|
|
|
print(to_log)
|
|
|
|
# Recurse in order to print whatever o is outputting, if anything
|
|
if o is not None:
|
|
o_lines = str(o).split("\n")
|
|
for line in o_lines:
|
|
self.log(line)
|
|
|
|
#
|
|
def eprint(*args, **kwargs):
|
|
|
|
print(*args, file=sys.stderr, **kwargs)
|
|
|
|
#
|
|
def parse_args(self):
|
|
|
|
#
|
|
print()
|
|
self.log("Parsing arguments")
|
|
a = 0
|
|
while a + 1 < len(sys.argv):
|
|
|
|
a += 1
|
|
|
|
#
|
|
arg = sys.argv[a]
|
|
|
|
#
|
|
valid_arg = False
|
|
if arg == "":
|
|
valid_arg = True
|
|
elif arg == "--full":
|
|
valid_arg = True
|
|
self.__force_full = True
|
|
elif arg == "--diff" or arg == "--differential":
|
|
valid_arg = True
|
|
self.__force_differential = True
|
|
elif arg == "--log-dir":
|
|
valid_arg = True
|
|
self.__log_dir = sys.argv[a + 1]
|
|
self.log("Found log dir: " + self.__log_dir)
|
|
a = a + 1
|
|
elif arg == "--source-dir":
|
|
valid_arg = True
|
|
self.__source_dirs.append(sys.argv[a + 1])
|
|
self.log("Found source dir: " + sys.argv[a + 1])
|
|
a = a + 1
|
|
elif arg == "--exclude":
|
|
valid_arg = True
|
|
self.__source_dir_excludes.append(sys.argv[a + 1])
|
|
self.log("Found exclude dir: " + sys.argv[a + 1])
|
|
a = a + 1
|
|
elif arg == "--source-mountpoint":
|
|
valid_arg = True
|
|
self.__source_mountpoint_demands.append(sys.argv[a + 1])
|
|
self.log("Found demanded source mountpoint: " + sys.argv[a + 1])
|
|
a += 1
|
|
elif arg == "--destination-dir":
|
|
valid_arg = True
|
|
self.__destination_dir_base = sys.argv[a + 1]
|
|
self.log("Found destination dir: " + self.__destination_dir_base)
|
|
a = a + 1
|
|
elif arg == "--remote-host":
|
|
valid_arg = True
|
|
self.__remote_host = sys.argv[a + 1]
|
|
self.log("Found remote host: " + self.__remote_host)
|
|
a = a + 1
|
|
elif arg == "--remote-user":
|
|
valid_arg = True
|
|
self.__remote_user = sys.argv[a + 1]
|
|
self.log("Found remote user: " + self.__remote_user)
|
|
a = a + 1
|
|
elif arg == "--ssh-key":
|
|
valid_arg = True
|
|
self.__ssh_key = sys.argv[a + 1]
|
|
self.log("Found ssh key: " + self.__ssh_key)
|
|
a = a + 1
|
|
|
|
#
|
|
if not valid_arg:
|
|
raise Exception("Invalid argument:", arg)
|
|
|
|
@staticmethod
|
|
def get_datetime_for_logging():
|
|
|
|
#
|
|
return datetime.datetime.now().strftime("%b %d %Y; %I%M%p")
|
|
|
|
@staticmethod
|
|
def get_datetime_for_filename():
|
|
|
|
#
|
|
return datetime.datetime.now().strftime('%Y-%b-%d_%I%M%p')
|
|
|
|
#
|
|
def demand_source_mountpoints(self):
|
|
|
|
for mountpoint_path in self.__source_mountpoint_demands:
|
|
if not os.path.ismount(mountpoint_path):
|
|
raise Exception("Demanded mountpoint is not mounted: " + str(mountpoint_path))
|
|
self.log("Verified mountpoint: " + mountpoint_path)
|
|
|
|
#
|
|
def is_using_ssh(self):
|
|
|
|
#
|
|
if (
|
|
self.__remote_host is not None
|
|
or self.__remote_user is not None
|
|
or self.__ssh_key is not None
|
|
):
|
|
return True
|
|
|
|
return False
|
|
|
|
#
|
|
def demand_ssh_config(self):
|
|
|
|
#
|
|
if self.is_using_ssh():
|
|
if self.__remote_host is None:
|
|
raise Exception("Please provide remote host")
|
|
if self.__remote_user is None:
|
|
raise Exception("Please provide remote user")
|
|
|
|
#
|
|
def demand_destination_directory_config(self):
|
|
|
|
#
|
|
if self.__destination_dir_base is None:
|
|
raise Exception("Please provide backup destination directory")
|
|
|
|
#
|
|
def does_destination_directory_exist(self, destination_path):
|
|
|
|
#
|
|
self.log("Trying to determine if destination path exists:" + destination_path)
|
|
|
|
# Local?
|
|
if not self.is_using_ssh():
|
|
self.log("Checking for local destination path")
|
|
if os.path.isdir(destination_path):
|
|
self.log("Local destination path exists")
|
|
return True
|
|
else:
|
|
self.log("Local destination path does not exist")
|
|
return False
|
|
|
|
#
|
|
self.log("Checking for remote destination path")
|
|
command = [
|
|
"[ -d " + destination_path + " ]"
|
|
]
|
|
|
|
#
|
|
code, stdout, stderr = self.execute_remote_ssh_command(command)
|
|
if code == 0:
|
|
self.log("Remote destination dir was found: " + destination_path)
|
|
return True
|
|
|
|
#
|
|
self.log("Remote dir didn't seem to exist: " + destination_path)
|
|
return False
|
|
|
|
#
|
|
def demand_destination_base_backup_directory(self):
|
|
|
|
#
|
|
self.demand_destination_directory_config()
|
|
|
|
#
|
|
destination_path = self.__destination_dir_base
|
|
|
|
#
|
|
if self.does_destination_directory_exist(destination_path) is False:
|
|
raise Exception("Backup destination directory doesn't exist: " + destination_path)
|
|
|
|
#
|
|
def does_full_backup_destination_directory_exist(self):
|
|
|
|
#
|
|
dir_path = self.make_full_backup_destination_path()
|
|
|
|
#
|
|
self.log("Trying to determine if Full backup destination directory exists:", dir_path)
|
|
return self.does_destination_directory_exist(dir_path)
|
|
|
|
#
|
|
def get_source_directories(self):
|
|
|
|
#
|
|
if len(self.__source_dirs) == 0:
|
|
raise Exception("No source directories specified")
|
|
|
|
return self.__source_dirs
|
|
|
|
#
|
|
def do_backup(self):
|
|
|
|
#
|
|
print()
|
|
self.log("Enter: do_backup")
|
|
|
|
# Source mountpoints must be mounted
|
|
self.demand_source_mountpoints()
|
|
|
|
# Remote base dir must exist
|
|
self.demand_destination_base_backup_directory()
|
|
|
|
raise Exception("Just testing")
|
|
|
|
# Forced full or differential by args?
|
|
if self.__force_full is True or self.__force_differential is True:
|
|
if self.__force_full is True:
|
|
self.log("Forcing full backup")
|
|
self.do_full_backup()
|
|
else:
|
|
self.log("Forcing differential backup")
|
|
self.do_differential_backup()
|
|
return
|
|
|
|
# Automatically choose full or differential
|
|
if self.does_full_backup_destination_directory_exist():
|
|
self.log("Automatically choosing differential backup, because full backup destination directory already exists")
|
|
self.do_differential_backup()
|
|
else:
|
|
self.log("Automatically choosing full backup, because full backup destination directory wasn't found")
|
|
self.do_full_backup()
|
|
|
|
#
|
|
def do_full_backup(self):
|
|
|
|
# Start args
|
|
args = []
|
|
|
|
# Get destination directory
|
|
destination_dir = self.make_full_backup_destination_path()
|
|
|
|
# Append source directories
|
|
args.extend(self.get_source_directories())
|
|
|
|
# Append remote destination directory
|
|
# args.append( self.__remote_user + "@" + self.__remote_host + ":" + remote_dir)
|
|
args.append(self.make_rsync_remote_destination_part(destination_dir))
|
|
|
|
# print("Args", str(args))
|
|
self.log("Destination dir:" + destination_dir)
|
|
|
|
self.execute_rsync(args)
|
|
|
|
#
|
|
def do_differential_backup(self):
|
|
|
|
# Start args
|
|
args = []
|
|
|
|
# Get directories
|
|
link_dest_dir = self.make_full_backup_destination_path()
|
|
destination_dir = self.make_remote_differential_backup_path()
|
|
self.ensure_destination_directory(destination_dir)
|
|
|
|
# Add link dest arg
|
|
args.append("--link-dest")
|
|
args.append(link_dest_dir)
|
|
|
|
# Append source directories
|
|
args.extend(self.get_source_directories())
|
|
|
|
# Append remote destination directory
|
|
# args.append( self.__remote_user + "@" + self.__remote_host + ":" + remote_dir)
|
|
args.append(self.make_rsync_remote_destination_part(destination_dir))
|
|
|
|
# print("Args", str(args))
|
|
self.log("Link destination dir:" + link_dest_dir)
|
|
self.log("Destination dir:" + destination_dir)
|
|
|
|
self.execute_rsync(args)
|
|
|
|
#
|
|
def make_log_directory_path(self):
|
|
|
|
#
|
|
log_dir = self.__log_dir
|
|
if log_dir is None:
|
|
self.log("No log directory specified; Defaulting to current working directory")
|
|
log_dir = "."
|
|
|
|
return log_dir
|
|
|
|
#
|
|
def make_log_path(self):
|
|
|
|
# Log dir
|
|
log_dir = self.make_log_directory_path()
|
|
|
|
# Path
|
|
log_path = os.path.join(log_dir, self.get_datetime_for_filename() + ".log")
|
|
|
|
return log_path
|
|
|
|
#
|
|
def make_full_backup_destination_path(self):
|
|
|
|
#
|
|
if self.__destination_dir_base is None:
|
|
raise Exception("No remote directory was specified")
|
|
|
|
#
|
|
return os.path.join(self.__destination_dir_base, "Full")
|
|
|
|
#
|
|
def make_remote_differential_backup_path(self):
|
|
|
|
#
|
|
if self.__destination_dir_base is None:
|
|
raise Exception("No remote directory was specified")
|
|
|
|
#
|
|
return os.path.join(self.__destination_dir_base, "Differential", self.get_datetime_for_filename())
|
|
|
|
#
|
|
def make_rsync_remote_destination_part(self, destination_dir):
|
|
|
|
#
|
|
part = ""
|
|
|
|
#
|
|
if self.__remote_host is not None:
|
|
if self.__remote_user is not None:
|
|
part += self.__remote_user + "@"
|
|
part += self.__remote_host + ":"
|
|
|
|
#
|
|
part += destination_dir
|
|
|
|
return part
|
|
|
|
@staticmethod
|
|
def ensure_local_directory(d):
|
|
|
|
#
|
|
if not os.path.exists(d):
|
|
os.makedirs(d)
|
|
|
|
#
|
|
def ensure_destination_directory(self, d):
|
|
|
|
#
|
|
if not self.does_destination_directory_exist(d):
|
|
|
|
#
|
|
self.log("Destination directory doesn't exist; Will create:" + d)
|
|
|
|
#
|
|
if self.is_using_ssh():
|
|
command = [
|
|
"mkdir",
|
|
"--parents",
|
|
d
|
|
]
|
|
self.execute_remote_ssh_command(command)
|
|
else:
|
|
os.makedirs(d)
|
|
|
|
#
|
|
def start_rsync_args(self):
|
|
|
|
#
|
|
self.ensure_local_directory(self.make_log_directory_path())
|
|
args = [
|
|
"rsync",
|
|
"--log-file", self.make_log_path(),
|
|
"--archive",
|
|
"--compress",
|
|
"--progress",
|
|
"--stats",
|
|
"--human-readable",
|
|
"--itemize-changes",
|
|
"--one-file-system",
|
|
"--delete",
|
|
"--delete-excluded"
|
|
]
|
|
|
|
#
|
|
for e in self.__source_dir_excludes:
|
|
args.append("--exclude")
|
|
args.append(e)
|
|
|
|
#
|
|
# args.append("--dry-run") # DEBUG !!!
|
|
|
|
return args
|
|
|
|
#
|
|
def start_rsync_environment_variables(self):
|
|
|
|
#
|
|
env = {}
|
|
|
|
#
|
|
if self.__ssh_key is not None or self.__quiet_ssh is True:
|
|
env["RSYNC_RSH"] = "ssh"
|
|
if self.__ssh_key is not None:
|
|
env["RSYNC_RSH"] += " -i " + shlex.quote(self.__ssh_key)
|
|
if self.__quiet_ssh is True:
|
|
env["RSYNC_RSH"] += " -q"
|
|
|
|
return env
|
|
|
|
#
|
|
def execute_remote_ssh_command(self, command):
|
|
|
|
#
|
|
self.demand_ssh_config()
|
|
|
|
#
|
|
args = list()
|
|
|
|
# ssh command
|
|
args.append("ssh")
|
|
|
|
# Quiet?
|
|
if self.__quiet_ssh is True:
|
|
args.append("-q")
|
|
|
|
# ssh key
|
|
if self.__ssh_key is not None:
|
|
args.append("-i")
|
|
args.append(self.__ssh_key)
|
|
|
|
# ssh user@host
|
|
args.append(self.__remote_user + "@" + self.__remote_host)
|
|
|
|
# Append the command
|
|
args.append("--")
|
|
if isinstance(command, str):
|
|
args.append(command)
|
|
elif isinstance(command, list):
|
|
args.extend(command)
|
|
else:
|
|
raise Exception("Unsupported command datatype")
|
|
|
|
# Spawn SSH in shell
|
|
# process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
process = subprocess.Popen(args)
|
|
# stdout, stderr = process.communicate()
|
|
process.communicate()
|
|
stdout = ""
|
|
stderr = ""
|
|
# print(stderr.decode())
|
|
|
|
return process.returncode, stdout, stderr
|
|
|
|
#
|
|
def execute_rsync(self, _args):
|
|
|
|
# Demand stuff
|
|
self.demand_destination_directory_config()
|
|
if self.is_using_ssh():
|
|
self.demand_ssh_config()
|
|
|
|
#
|
|
args = self.start_rsync_args()
|
|
args.extend(_args)
|
|
# print(str(args))
|
|
|
|
#
|
|
env = self.start_rsync_environment_variables()
|
|
|
|
#
|
|
# print("Debug -> Want to execute Rsync")
|
|
# print("Args:", str(args))
|
|
# print("Env:", str(env))
|
|
# return (0, "", "")
|
|
|
|
# Spawn Rsync in shell
|
|
# process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
|
|
process = subprocess.Popen(args, env=env)
|
|
# stdout, stderr = process.communicate()
|
|
process.communicate()
|
|
# self.eprint(stderr.decode())
|
|
|
|
stdout = ""
|
|
stderr = ""
|
|
|
|
return process.returncode, stdout, stderr
|
|
|
|
|
|
def main():
|
|
|
|
b = MikesBackup()
|
|
b.do_backup()
|
|
|
|
|
|
#
|
|
if __name__ == "__main__":
|
|
|
|
#
|
|
main()
|