26 Commits

Author SHA1 Message Date
165787f819 Changed to lowercase "full" and "differential" for dir creation 2019-08-06 15:28:49 -07:00
4b0ea6a411 Bugfix -> Backups to a local remote stopped working due to determine_rsync_backup_link_destination_path()
So like ... fixed it. And stuff.
2019-08-06 15:27:44 -07:00
74c347ec6a Changed a word 2019-08-06 15:27:06 -07:00
85bb2d7290 Only limit rsync to one file systen if no --source-mountpoint demands have been set 2019-08-05 06:23:11 -07:00
b11d69c957 Trying to improve logging a tad 2019-08-05 03:36:23 -07:00
b979b6196f Enforce that --source-dir can only be specified once 2019-08-05 03:20:45 -07:00
9d66f59113 When renaming temp differential dir to final, add support for when we're using ssh 2019-08-05 03:09:41 -07:00
54321e7ec1 MikesBackup::determine_rsync_backup_link_destination_path() Bugfix -> Make sure remote differential base path exists before trying to "ls" it 2019-08-05 03:03:13 -07:00
3622d365f1 Add new CLI parameter to disable incremental backups (ie: only use full as the link target; never a previous differential) 2019-08-05 01:30:27 -07:00
4a0d4abc54 After successful full backup, delete the entire diff backup directory, since it should technically all be out of date 2019-08-05 01:24:39 -07:00
6b70a22b52 Add some verbosity to param parsing 2019-08-05 01:23:45 -07:00
58aec3f911 MikesBackup::execute_rsync() - Raise an exception if we don't get a successful return code 2019-08-05 01:23:26 -07:00
9106924993 Ignore the "in progress" diff directory when considering a link target, derp 2019-08-05 01:22:26 -07:00
1d3e08e804 Implemented an "in progress" dir for differentials that only gets renamed upon success 2019-08-05 01:06:54 -07:00
fca18717fb Upgrade the logging methods 2019-08-05 01:02:48 -07:00
785c9e4c96 Updated README 2019-08-05 00:29:39 -07:00
0399ffde87 Upgrades and tweaks
- Added "include" feature from rsync
- Made some constants
- When making a diff, the most recently created directory will be used as the link, to save time and space (both full and diff directories are considered now)
-
2019-08-05 00:29:27 -07:00
8bb53d8a55 Add __str__() method to just print settings and ahstuffff 2019-08-04 18:59:32 -07:00
af6c37555c Tweak CLI parsing because it was wonky 2019-08-04 18:59:09 -07:00
8613bb8fce Slight upgrade to logging 2019-08-04 18:35:24 -07:00
f5b31f1dbb Added support for required source mountpoints 2019-08-04 18:21:48 -07:00
99a3bca3ba Appease Pycharm's incessant complaints 2019-08-04 18:04:45 -07:00
ee2e1af737 Ignore pycharm 2019-08-04 17:51:07 -07:00
5f51ff9756 Noop to test hook 2019-08-04 02:05:18 -07:00
933e3176a7 Noop to test hook 2019-08-04 01:58:55 -07:00
9eb1046351 No op to test hook 2019-08-04 01:58:19 -07:00
3 changed files with 558 additions and 219 deletions

4
.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
#
.idea

View File

@ -14,13 +14,17 @@ This script is really just a wrapper around rsync's beautiful functionality. It
## Command Line Arguments ## Command Line Arguments
* ```--full``` Forces the script to run a *full* backup * ```--full``` Forces the script to run a *full* backup
* ```--differential``` Forces the script to run a *differential* backup * ```--differential``` Forces the script to run a *differential* backup
* ```--diff``` Same as ```---differential```
* ```--no-incremental``` Always force differentials to link back to the *full* backup, and not the most recent *differential*
* ```--log-dir <directory>``` Let's you set the log output directory * ```--log-dir <directory>``` Let's you set the log output directory
* ```--source-dir <directory>``` Specifies the local source directory * ```--source-dir <directory>``` Specifies the local source directory
* ```--include <directory>``` Specifies another local source directory to include in the backup
* ```--source-mountpoint <directory>``` Make sure a local mountpoint is mounted before continuing
* ```--destination-dir <directory>``` Specifies the backup destination directory * ```--destination-dir <directory>``` Specifies the backup destination directory
* ```--exclude <dir>``` Specifies a source directory to exclude (can be passed multiple times)
* ```--remote-host <hostname>``` Specifies the remote host, if your backup destination is an SSH server * ```--remote-host <hostname>``` Specifies the remote host, if your backup destination is an SSH server
* ```--remote-user <username>``` Specifies the remote username to use, if your backup destination is an SSH server * ```--remote-user <username>``` Specifies the remote username to use, if your backup destination is an SSH server
* ```--ssh-key <path to key>``` Specifies the local SSH key to use for authentication, if your backup destination is an SSH server * ```--ssh-key <path to key>``` Specifies the local SSH key to use for authentication, if your backup destination is an SSH server
* ```--exclude <dir>``` Specifies a source directory to exclude (can be passed multiple times)
Note that ```--remote-host```, ```--remote-user```, and ```--ssh-key``` are only needed if your backup destination is a remote SSH server. You may omit all three if the destination is a locally mounted folder. Note that ```--remote-host```, ```--remote-user```, and ```--ssh-key``` are only needed if your backup destination is a remote SSH server. You may omit all three if the destination is a locally mounted folder.

View File

@ -1,326 +1,513 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# #
import datetime import datetime
import os import os
import re
import shlex import shlex
import subprocess import subprocess
import sys import sys
# #
class MikesBackup: class MikesBackup:
# #
_log_dir = None __log_dir = None
_remote_host = None
_remote_user = None
_destination_dir_base = None
#
_ssh_key = None
_quiet_ssh = True
#
_source_dirs = []
_source_dir_excludes = []
#
_force_full = False
_force_differential = False
__remote_host = None
__remote_user = None
__destination_dir_base = None
__source_dir = None
__source_dir_includes = []
__source_dir_excludes = []
__source_mountpoint_demands = []
__ssh_key = None
__quiet_ssh = True
__force_full = False
__force_differential = False
__no_incremental = False
__log_file_handle = None
CONST_FULL_DIRECTORY_NAME = "full"
CONST_DIFFERENTIAL_DIRECTORY_NAME = "differential"
CONST_DIFFERENTIAL_IN_PROGRESS_DIRECTORY_NAME = "IN-PROGRESS"
# #
def __init__(self): def __init__(self):
self.ParseArgs()
self.parse_args()
#
def __del__(self):
self.close_log_file()
#
def __str__(self):
s = ""
s += "MikesBackup Class Instance"
s += "\nLog Dir: " + str(self.__log_dir)
s += "\nRemote Host: " + str(self.__remote_host)
s += "\nRemote User: " + str(self.__remote_user)
s += "\nDestination Dir Base: " + str(self.__destination_dir_base)
s += "\nSource Dir (Main): " + str(self.__source_dir)
s += "\nSource Dirs (Includes): " + str(self.__source_dir_includes)
s += "\nSource Dirs (Excludes): " + str(self.__source_dir_excludes)
s += "\nSource Mountpoint Demands: " + str(self.__source_mountpoint_demands)
s += "\nSSH Key: " + str(self.__ssh_key)
s += "\nQuiet SSH: " + str(self.__quiet_ssh)
s += "\nForce Full Backup: " + str(self.__force_full)
s += "\nForce Differential: " + str(self.__force_differential)
s += "\nDisallow Incremental: " + str(self.__no_incremental)
return s
#
def log(self, s, o=None):
the_date = self.get_datetime_for_logging()
to_log = "[MikesBackup][" + the_date + "] " + s
# Print the log line
print(to_log)
# Append the log line to the log file
f = self.open_log_file()
if f:
f.write(to_log + "\n")
# Recurse in order to print whatever o is outputting, if anything
if o is not None:
o_lines = str(o).split("\n")
for line in o_lines:
self.log(line)
#
def open_log_file(self):
if self.__log_file_handle:
return self.__log_file_handle
log_file_path = self.make_log_path()
if log_file_path:
self.__log_file_handle = open(log_file_path, "w")
return self.__log_file_handle
#
def close_log_file(self):
if self.__log_file_handle:
self.__log_file_handle.close()
self.__log_file_handle = None
# #
def eprint(*args, **kwargs): def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs) print(*args, file=sys.stderr, **kwargs)
# #
def log(): def parse_args(self):
pass
#
def ParseArgs(self):
# Simple mapped args
args_map = {
"--log-dir" : "_log_dir",
"--remote-host" : "_remote_host",
"--remote-user" : "_remote_user",
"--destination-dir" : "_destination_dir_base",
"--ssh-key" : "_ssh_key"
}
# #
print() print()
print ("Parsing arguments") self.log("Parsing arguments")
a = 1 a = 0
while a < len(sys.argv): while a + 1 < len(sys.argv):
a += 1
# #
arg = sys.argv[a] arg = sys.argv[a]
# #
valid_arg = False valid_arg = False
for arg_name in args_map:
if arg == arg_name:
valid_arg = True
self_var_name = args_map[arg_name]
self_var_value = sys.argv[ a + 1 ]
self.__dict__[self_var_name] = self_var_value
print("Found argument \"", arg_name, "\" ==>", self_var_value)
a = a + 1
break
if arg == "": if arg == "":
valid_arg = True valid_arg = True
elif arg == "--full": elif arg == "--full":
valid_arg = True valid_arg = True
self._force_full = True self.__force_full = True
self.log("Forcing a full backup")
elif arg == "--diff" or arg == "--differential": elif arg == "--diff" or arg == "--differential":
valid_arg = True valid_arg = True
self._force_differential = True self.__force_differential = True
self.log("Forcing a differential backup")
elif arg == "--no-incremental":
valid_arg = True
self.__no_incremental = True
self.log("Disallowing incremental backups (differentials will only link back to full)")
elif arg == "--log-dir":
valid_arg = True
self.__log_dir = sys.argv[a + 1]
self.log("Found log dir: " + self.__log_dir)
a = a + 1
elif arg == "--source-dir": elif arg == "--source-dir":
valid_arg = True valid_arg = True
self._source_dirs.append( sys.argv[ a + 1 ] ) if self.__source_dir:
print("Found source dir:", sys.argv[ a + 1 ]) raise Exception("--source-dir can only be used once")
self.__source_dir = sys.argv[a + 1]
self.log("Found source dir: " + sys.argv[a + 1])
a = a + 1
elif arg == "--include":
valid_arg = True
self.__source_dir_includes.append(sys.argv[a + 1])
self.log("Found additional source dir include: " + sys.argv[a + 1])
a = a + 1
elif arg == "--source-mountpoint":
valid_arg = True
self.__source_mountpoint_demands.append(sys.argv[a + 1])
self.log("Found required source mountpoint: " + sys.argv[a + 1])
a += 1
elif arg == "--destination-dir":
valid_arg = True
self.__destination_dir_base = sys.argv[a + 1]
self.log("Found destination dir: " + self.__destination_dir_base)
a = a + 1 a = a + 1
elif arg == "--exclude": elif arg == "--exclude":
valid_arg = True valid_arg = True
self._source_dir_excludes.append( sys.argv[ a + 1 ] ) self.__source_dir_excludes.append(sys.argv[a + 1])
print("Found exclude dir:", sys.argv[ a + 1 ]) self.log("Found exclude dir: " + sys.argv[a + 1])
a = a + 1 a = a + 1
elif arg == "--remote-host":
valid_arg = True
self.__remote_host = sys.argv[a + 1]
self.log("Found remote host: " + self.__remote_host)
a = a + 1
elif arg == "--remote-user":
valid_arg = True
self.__remote_user = sys.argv[a + 1]
self.log("Found remote user: " + self.__remote_user)
a = a + 1
elif arg == "--ssh-key":
valid_arg = True
self.__ssh_key = sys.argv[a + 1]
self.log("Found ssh key: " + self.__ssh_key)
a = a + 1 a = a + 1
# #
if not valid_arg: if not valid_arg:
raise Exception("Invalid argument:", arg); raise Exception("Invalid argument:", arg)
@staticmethod
def get_datetime_for_logging():
# #
def GetDatetimeForFilename(self): return datetime.datetime.now().strftime("%b %d %Y; %I%M%p")
@staticmethod
def get_datetime_for_filename():
# #
return datetime.datetime.now().strftime('%Y-%b-%d_%I%M%p') return datetime.datetime.now().strftime('%Y-%b-%d_%I%M%p')
# #
def IsUsingSSH(self): def is_using_source_mountpoints(self):
return len(self.__source_mountpoint_demands) > 0
# #
if self._remote_host != None or self._remote_user != None or self._ssh_key != None: def demand_source_mountpoints(self):
for mountpoint_path in self.__source_mountpoint_demands:
if not os.path.ismount(mountpoint_path):
raise Exception("Required mountpoint is not mounted: " + str(mountpoint_path))
self.log("Verified mountpoint: " + mountpoint_path)
#
def is_using_ssh(self):
#
if (
self.__remote_host is not None
or self.__remote_user is not None
or self.__ssh_key is not None
):
return True return True
return False return False
# #
def DemandSSHConfig(self): def demand_ssh_config(self):
# #
if self.IsUsingSSH(): if self.is_using_ssh():
if self._remote_host == None: if self.__remote_host is None:
raise Exception("Please provide remote host") raise Exception("Please provide remote host")
if self._remote_user == None: if self.__remote_user is None:
raise Exception("Please provide remote user") raise Exception("Please provide remote user")
# #
def DemandDestinationDirectoryConfig(self): def demand_source_directory_config(self):
# #
if self._destination_dir_base == None: if self.__source_dir is None:
raise Exception("Please provide a source directory")
#
def demand_destination_directory_config(self):
#
if self.__destination_dir_base is None:
raise Exception("Please provide backup destination directory") raise Exception("Please provide backup destination directory")
# #
def DoesDestinationDirectoryExist(self, destination_path): def does_destination_directory_exist(self, destination_path):
# #
print("Trying to determine if destination path exists:", destination_path) self.log("Trying to determine if destination path exists:" + destination_path)
# Local? # Local?
if not self.IsUsingSSH(): if not self.is_using_ssh():
print("Checking for local destination path") self.log("Checking for local destination path")
if os.path.isdir(destination_path): if os.path.isdir(destination_path):
print("Local destination path exists") self.log("Local destination path exists")
return True return True
else: else:
print("Local destination path does not exist") self.log("Local destination path does not exist")
return False return False
# #
print("Checking for remote destination path") self.log("Checking for remote destination path")
command = [ command = [
"[ -d " + destination_path + " ]" "[ -d " + destination_path + " ]"
] ]
# #
code, stdout, stderr = self.ExecuteRemoteSSHCommand(command) code, stdout, stderr = self.execute_remote_ssh_command(command)
if code == 0: if code == 0:
print("Remote destination dir was found: " + destination_path) self.log("Remote destination dir was found: " + destination_path)
return True return True
# #
print("Remote dir didn't seem to exist: " + destination_path) self.log("Remote dir didn't seem to exist: " + destination_path)
return False return False
# #
def DemandDestinationBaseBackupDirectory(self): def demand_destination_base_backup_directory(self):
# #
self.DemandDestinationDirectoryConfig() self.demand_destination_directory_config()
# #
destination_path = self._destination_dir_base destination_path = self.__destination_dir_base
# #
if self.DoesDestinationDirectoryExist(destination_path) == False: if self.does_destination_directory_exist(destination_path) is False:
raise Exception("Backup destination directory doesn't exist: " + destination_path) raise Exception("Backup destination directory doesn't exist: " + destination_path)
# #
def DoesFullBackupDestinationDirectoryExist(self): def does_full_backup_destination_directory_exist(self):
# #
dir_path = self.MakeFullBackupDestinationPath() dir_path = self.make_full_backup_destination_path()
# #
print("Trying to determine if Full backup destination directory exists:", dir_path) self.log("Trying to determine if Full backup destination directory exists:", dir_path)
return self.DoesDestinationDirectoryExist(dir_path) return self.does_destination_directory_exist(dir_path)
# #
def GetSourceDirectories(self): def do_backup(self):
#
if len(self._source_dirs) == 0:
raise Exception("No source directories specified")
return self._source_dirs
#
def DoBackup(self):
# #
print() print()
print("Enter: DoBackup") self.log("Enter: do_backup")
# Source mountpoints must be mounted
self.demand_source_mountpoints()
# Remote base dir must exist # Remote base dir must exist
self.DemandDestinationBaseBackupDirectory() self.demand_destination_base_backup_directory()
# Forced full or differential by args? # Forced full or differential by args?
if self._force_full == True or self._force_differential == True: if self.__force_full is True or self.__force_differential is True:
if self._force_full == True: if self.__force_full is True:
print("Forcing full backup") self.log("Forcing full backup")
self.DoFullBackup() self.do_full_backup()
else: else:
print("Forcing differential backup") self.log("Forcing differential backup")
self.DoDifferentialBackup() self.do_differential_backup()
return return
# Automatically choose full or differential # Automatically choose full or differential
if self.DoesFullBackupDestinationDirectoryExist(): if self.does_full_backup_destination_directory_exist():
print("Automatically choosing differential backup, because full backup destination directory already exists") self.log("Automatically choosing differential backup, because full backup destination directory already exists")
self.DoDifferentialBackup() self.do_differential_backup()
else: else:
print("Automatically choosing full backup, because full backup destination directory wasn't found") self.log("Automatically choosing full backup, because full backup destination directory wasn't found")
self.DoFullBackup() self.do_full_backup()
# #
def DoFullBackup(self): def do_full_backup(self):
# Start args # Start args
args = [] args = []
# Get destination directory # Get destination directory
destination_dir = self.MakeFullBackupDestinationPath() destination_dir = self.make_full_backup_destination_path()
# Append source directories # Append source directory
args.extend(self.GetSourceDirectories()) args.append(self.make_rsync_source_directory_part())
# Append remote destination directory # Append remote destination directory
#args.append( self._remote_user + "@" + self._remote_host + ":" + remote_dir) # args.append( self.__remote_user + "@" + self.__remote_host + ":" + remote_dir)
args.append( self.MakeRsyncRemoteDestinationPart(destination_dir) ) args.append(self.make_rsync_remote_destination_part(destination_dir))
# print("Args", str(args)) # print("Args", str(args))
print("Destination dir:", destination_dir) self.log("Destination dir:" + destination_dir)
self.ExecuteRsync(args) self.execute_rsync(args)
self.log("Rsync seems to have finished successfully")
self.log("Because a full backup has succeeded, will now delete any differential backups")
self.execute_remote_ssh_command(
[
"rm",
"-rfv",
self.make_remote_differential_backup_path_base()
])
self.log("Finished deleting old differentials")
# #
def DoDifferentialBackup(self): def do_differential_backup(self):
# Start args # Start args
args = [] args = []
# Get directories # Get directories
link_dest_dir = self.MakeFullBackupDestinationPath() link_dest_dir = self.determine_rsync_backup_link_destination_path()
destination_dir = self.MakeRemoteDifferentialBackupPath() destination_dir_in_progress = self.make_remote_differential_in_progress_backup_path()
self.EnsureDestinationDirectory(destination_dir) destination_dir_final = self.make_remote_differential_backup_path()
self.ensure_destination_directory(destination_dir_in_progress)
# Add link dest arg # Add link dest arg?
if link_dest_dir:
args.append("--link-dest") args.append("--link-dest")
args.append(link_dest_dir) args.append(link_dest_dir)
# Append source directories # Append source directory
args.extend(self.GetSourceDirectories()) args.append(self.make_rsync_source_directory_part())
# Append remote destination directory # Append remote destination directory
#args.append( self._remote_user + "@" + self._remote_host + ":" + remote_dir) args.append(self.make_rsync_remote_destination_part(destination_dir_in_progress))
args.append( self.MakeRsyncRemoteDestinationPart(destination_dir) )
#print("Args", str(args)) self.log("Link destination dir: " + link_dest_dir)
print("Link destination dir:", link_dest_dir) self.log("Destination dir: " + destination_dir_in_progress)
print("Destination dir:", destination_dir)
self.ExecuteRsync(args) self.execute_rsync(args)
self.log("Rsync seems to have finished successfully")
self.log("Renaming temporary directory")
self.log("Old: " + destination_dir_in_progress)
self.log("New: " + destination_dir_final)
if self.is_using_ssh():
return_code, stdout, stderr = self.execute_remote_ssh_command([
"mv",
destination_dir_in_progress,
destination_dir_final
])
if return_code != 0:
raise Exception("Failed to move temporary diff directory to its final home")
else:
os.rename(destination_dir_in_progress, destination_dir_final)
self.log("Rename was successful")
# #
def MakeLogDirectoryPath(self): def make_log_directory_path(self):
# #
log_dir = self._log_dir log_dir = self.__log_dir
if log_dir == None: if log_dir is None:
print("No log directory specified; Defaulting to current working directory") print("No log directory specified; Won't log")
log_dir = "." return None
return log_dir return log_dir
# #
def MakeLogPath(self): def make_log_path(self):
# Log dir # Log dir
log_dir = self.MakeLogDirectoryPath() log_dir = self.make_log_directory_path()
if not log_dir:
return None
# Path # Path
log_path = os.path.join(log_dir, self.GetDatetimeForFilename() + ".log") log_path = os.path.join(log_dir, self.get_datetime_for_filename() + ".log")
return log_path return log_path
# #
def MakeFullBackupDestinationPath(self): def make_full_backup_destination_path(self):
# #
if self._destination_dir_base == None: if self.__destination_dir_base is None:
raise Exception("No remote directory was specified") raise Exception("No remote directory was specified")
# #
return os.path.join(self._destination_dir_base, "Full") return os.path.join(self.__destination_dir_base, self.CONST_FULL_DIRECTORY_NAME)
# #
def MakeRemoteDifferentialBackupPath(self): def make_remote_differential_backup_path_base(self):
# #
if self._destination_dir_base == None: if self.__destination_dir_base is None:
raise Exception("No remote directory was specified") raise Exception("No remote directory was specified")
# return os.path.join(self.__destination_dir_base, self.CONST_DIFFERENTIAL_DIRECTORY_NAME)
return os.path.join(self._destination_dir_base, "Differential", self.GetDatetimeForFilename())
# #
def MakeRsyncRemoteDestinationPart(self, destination_dir): def make_remote_differential_in_progress_backup_path(self):
diff_path_base = self.make_remote_differential_backup_path_base()
return os.path.join(diff_path_base, self.CONST_DIFFERENTIAL_IN_PROGRESS_DIRECTORY_NAME)
#
def make_remote_differential_backup_path(self):
diff_path_base = self.make_remote_differential_backup_path_base()
#
return os.path.join(diff_path_base, self.get_datetime_for_filename())
#
def make_rsync_source_includes_part(self):
args = []
for d in self.__source_dir_includes:
args.append("--include")
args.append(d)
return args
#
def make_rsync_source_directory_part(self):
self.demand_source_directory_config()
#
return self.__source_dir
#
def make_rsync_remote_destination_part(self, destination_dir):
# #
part = "" part = ""
# #
if self._remote_host != None: if self.__remote_host is not None:
if self._remote_user != None: if self.__remote_user is not None:
part += self._remote_user + "@" part += self.__remote_user + "@"
part += self._remote_host + ":" part += self.__remote_host + ":"
# #
part += destination_dir part += destination_dir
@ -328,53 +515,157 @@ class MikesBackup:
return part return part
# #
def EnsureLocalDirectory(self, d): def determine_rsync_backup_link_destination_path(self):
self.demand_destination_directory_config()
self.log("Begin trying to determine which previous backup path to use as link")
#
newest_path = None
newest_path_date = None
# Pattern to parse the 'ls' command
pattern = re.compile(
""".*(?P<date>[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9} .{5}) (?P<name>.+)$""",
re.MULTILINE
)
# Get listing info for the full path
destination_path_full = self.make_full_backup_destination_path()
args_full_destination_path_ls = [
"ls",
"-l",
"-c",
"--all",
"--full-time",
destination_path_full
]
if self.is_using_ssh():
return_code, stdout, stderr = self.execute_remote_ssh_command(args_full_destination_path_ls)
else:
return_code, stdout, stderr = self.execute_command(args_full_destination_path_ls)
if return_code != 0:
raise Exception("Failed to get listing info for base destination directory")
for match in pattern.finditer(stdout):
name = match.group("name")
date = match.group("date")
if name == ".":
self.log("Start by assuming \"Full\" is the most recent backup: " + destination_path_full)
self.log("; With a date of " + date)
newest_path = destination_path_full
newest_path_date = date
break
if not newest_path:
self.log("Didn't find a \"Full\" backup on remote")
if self.__no_incremental:
self.log("Incremental backups are disabled; Won't consider any differential directories for the link target")
return newest_path
# Get listing info for all differential directories
differential_path_base = self.make_remote_differential_backup_path_base()
self.ensure_destination_directory(differential_path_base)
args_differential_destination_path_ls = [
"ls",
"-l",
"-c",
"--all",
"--full-time",
differential_path_base
]
if self.is_using_ssh():
return_code, stdout, stderr = self.execute_remote_ssh_command(args_differential_destination_path_ls)
else:
return_code, stdout, stderr = self.execute_command(args_differential_destination_path_ls)
if return_code != 0:
raise Exception("Failed to get listing info for destination differential base directory")
# Look for the most recent differential directory
# (must be newer than the Full directory too)
for match in pattern.finditer(stdout):
name = match.group("name")
date = match.group("date")
if name == "." or name == ".." or name == self.CONST_DIFFERENTIAL_IN_PROGRESS_DIRECTORY_NAME:
continue
if newest_path is None or date > newest_path_date:
self.log("Found a newer differential backup: " + name + "; " + date)
newest_path = os.path.join(differential_path_base, name)
newest_path_date = date
else:
self.log("Not newer: " + name + "; " + date)
#
self.log("Newest backup path is: " + newest_path)
self.log("; With a date of: " + newest_path_date)
return newest_path
@staticmethod
def ensure_local_directory(d):
# #
if not os.path.exists(d): if not os.path.exists(d):
os.makedirs(d) os.makedirs(d)
# #
def EnsureDestinationDirectory(self, d): def ensure_destination_directory(self, d):
# #
if not self.DoesDestinationDirectoryExist(d): if not self.does_destination_directory_exist(d):
# #
print("Destination directory doesn't exist; Will create:", d) self.log("Destination directory doesn't exist; Will create:" + d)
# #
if self.IsUsingSSH(): if self.is_using_ssh():
command = [ command = [
"mkdir", "mkdir",
"--parents", "--parents",
d d
] ]
self.ExecuteRemoteSSHCommand(command) self.execute_remote_ssh_command(command)
else: else:
os.makedirs(d) os.makedirs(d)
# #
def StartRsyncArgs(self): def start_rsync_args(self):
# #
self.EnsureLocalDirectory(self.MakeLogDirectoryPath())
args = [ args = [
"rsync", "rsync",
"--log-file", self.MakeLogPath(),
"--archive", "--archive",
"--compress", "--compress",
"--progress", "--progress",
"--stats", "--stats",
"--human-readable", "--human-readable",
"--itemize-changes", "--itemize-changes",
"--one-file-system",
"--delete", "--delete",
"--delete-excluded" "--delete-excluded"
] ]
log_dir = self.make_log_directory_path()
log_path = self.make_log_path()
if log_dir and log_path:
self.ensure_local_directory(log_dir)
args.append("--log-file")
args.append(log_path)
# Only allow recursion into multiple file systems
# if any mountpoints were specified
if not self.is_using_source_mountpoints():
args.append("--one-file-system")
# #
for e in self._source_dir_excludes: for i in self.__source_dir_includes:
args.append("--include")
args.append(i)
#
for e in self.__source_dir_excludes:
args.append("--exclude") args.append("--exclude")
args.append(e) args.append(e)
@ -384,44 +675,67 @@ class MikesBackup:
return args return args
# #
def StartRsyncEnvironmentalVariables(self): def start_rsync_environment_variables(self):
# #
env = {} env = {}
# #
if self._ssh_key != None or self._quiet_ssh == True: if self.__ssh_key is not None or self.__quiet_ssh is True:
env["RSYNC_RSH"] = "ssh" env["RSYNC_RSH"] = "ssh"
if self._ssh_key != None: if self.__ssh_key is not None:
env["RSYNC_RSH"] += " -i " + shlex.quote(self._ssh_key) env["RSYNC_RSH"] += " -i " + shlex.quote(self.__ssh_key)
if self._quiet_ssh == True: if self.__quiet_ssh is True:
env["RSYNC_RSH"] += " -q" env["RSYNC_RSH"] += " -q"
return env return env
# @staticmethod
def ExecuteRemoteSSHCommand(self, command): def execute_command(command):
# #
self.DemandSSHConfig() args = list()
# Append the command
if isinstance(command, str):
args.append(command)
elif isinstance(command, list):
args.extend(command)
else:
raise Exception("Unsupported command datatype")
# Spawn
process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
stdout = stdout.decode()
stderr = stderr.decode()
return process.returncode, stdout, stderr
# #
args = [] def execute_remote_ssh_command(self, command):
#
self.demand_ssh_config()
#
args = list()
# ssh command # ssh command
args.append("ssh") args.append("ssh")
# Quiet? # Quiet?
if self._quiet_ssh == True: if self.__quiet_ssh is True:
args.append("-q") args.append("-q")
# ssh key # ssh key
if self._ssh_key != None: if self.__ssh_key is not None:
args.append("-i") args.append("-i")
args.append(self._ssh_key) args.append(self.__ssh_key)
# ssh user@host # ssh user@host
args.append(self._remote_user + "@" + self._remote_host) args.append(self.__remote_user + "@" + self.__remote_host)
# Append the command # Append the command
args.append("--") args.append("--")
@ -432,32 +746,41 @@ class MikesBackup:
else: else:
raise Exception("Unsupported command datatype") raise Exception("Unsupported command datatype")
# Spawn SSH in shell # Spawn
#process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
process = subprocess.Popen(args) # process = subprocess.Popen(args)
#stdout, stderr = process.communicate() stdout, stderr = process.communicate()
process.communicate()
stdout = "" stdout = stdout.decode()
stderr = "" stderr = stderr.decode()
# process.communicate()
# stdout = ""
# stderr = ""
# print(stderr.decode()) # print(stderr.decode())
return (process.returncode, stdout, stderr) return process.returncode, stdout, stderr
# #
def ExecuteRsync(self, _args): def execute_rsync(self, _args):
# Demand stuff # Demand stuff
self.DemandDestinationDirectoryConfig() self.demand_source_directory_config()
if self.IsUsingSSH(): self.demand_destination_directory_config()
self.DemandSSHConfig() if self.is_using_ssh():
self.demand_ssh_config()
# #
args = self.StartRsyncArgs() args = self.start_rsync_args()
args.extend(_args) args.extend(_args)
# print(str(args)) # print(str(args))
# #
env = self.StartRsyncEnvironmentalVariables() env = self.start_rsync_environment_variables()
#
self.log("Executing rsync with the following arguments:", args)
self.log("; And the following environment:", env)
# #
# print("Debug -> Want to execute Rsync") # print("Debug -> Want to execute Rsync")
@ -475,16 +798,24 @@ class MikesBackup:
stdout = "" stdout = ""
stderr = "" stderr = ""
return (process.returncode, stdout, stderr) # Check return code
# 0 = Success
# 24 = Source files vanished
return_code = process.returncode
if return_code != 0 and return_code != 24:
raise Exception("Rsync seems to have failed somehow! Got return code: " + str(return_code))
return return_code, stdout, stderr
def main():
b = MikesBackup()
b.do_backup()
# #
if __name__ == "__main__": if __name__ == "__main__":
# #
b = MikesBackup() main()
b.DoBackup()