Compare commits
26 Commits
Author | SHA1 | Date | |
---|---|---|---|
165787f819 | |||
4b0ea6a411 | |||
74c347ec6a | |||
85bb2d7290 | |||
b11d69c957 | |||
b979b6196f | |||
9d66f59113 | |||
54321e7ec1 | |||
3622d365f1 | |||
4a0d4abc54 | |||
6b70a22b52 | |||
58aec3f911 | |||
9106924993 | |||
1d3e08e804 | |||
fca18717fb | |||
785c9e4c96 | |||
0399ffde87 | |||
8bb53d8a55 | |||
af6c37555c | |||
8613bb8fce | |||
f5b31f1dbb | |||
99a3bca3ba | |||
ee2e1af737 | |||
5f51ff9756 | |||
933e3176a7 | |||
9eb1046351 |
4
.gitignore
vendored
Normal file
4
.gitignore
vendored
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
|
||||||
|
#
|
||||||
|
.idea
|
||||||
|
|
@ -14,13 +14,17 @@ This script is really just a wrapper around rsync's beautiful functionality. It
|
|||||||
## Command Line Arguments
|
## Command Line Arguments
|
||||||
* ```--full``` Forces the script to run a *full* backup
|
* ```--full``` Forces the script to run a *full* backup
|
||||||
* ```--differential``` Forces the script to run a *differential* backup
|
* ```--differential``` Forces the script to run a *differential* backup
|
||||||
|
* ```--diff``` Same as ```---differential```
|
||||||
|
* ```--no-incremental``` Always force differentials to link back to the *full* backup, and not the most recent *differential*
|
||||||
* ```--log-dir <directory>``` Let's you set the log output directory
|
* ```--log-dir <directory>``` Let's you set the log output directory
|
||||||
* ```--source-dir <directory>``` Specifies the local source directory
|
* ```--source-dir <directory>``` Specifies the local source directory
|
||||||
|
* ```--include <directory>``` Specifies another local source directory to include in the backup
|
||||||
|
* ```--source-mountpoint <directory>``` Make sure a local mountpoint is mounted before continuing
|
||||||
* ```--destination-dir <directory>``` Specifies the backup destination directory
|
* ```--destination-dir <directory>``` Specifies the backup destination directory
|
||||||
|
* ```--exclude <dir>``` Specifies a source directory to exclude (can be passed multiple times)
|
||||||
* ```--remote-host <hostname>``` Specifies the remote host, if your backup destination is an SSH server
|
* ```--remote-host <hostname>``` Specifies the remote host, if your backup destination is an SSH server
|
||||||
* ```--remote-user <username>``` Specifies the remote username to use, if your backup destination is an SSH server
|
* ```--remote-user <username>``` Specifies the remote username to use, if your backup destination is an SSH server
|
||||||
* ```--ssh-key <path to key>``` Specifies the local SSH key to use for authentication, if your backup destination is an SSH server
|
* ```--ssh-key <path to key>``` Specifies the local SSH key to use for authentication, if your backup destination is an SSH server
|
||||||
* ```--exclude <dir>``` Specifies a source directory to exclude (can be passed multiple times)
|
|
||||||
|
|
||||||
Note that ```--remote-host```, ```--remote-user```, and ```--ssh-key``` are only needed if your backup destination is a remote SSH server. You may omit all three if the destination is a locally mounted folder.
|
Note that ```--remote-host```, ```--remote-user```, and ```--ssh-key``` are only needed if your backup destination is a remote SSH server. You may omit all three if the destination is a locally mounted folder.
|
||||||
|
|
||||||
|
701
mikes-backup
701
mikes-backup
@ -1,326 +1,513 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
|
||||||
#
|
#
|
||||||
import datetime
|
import datetime
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import shlex
|
import shlex
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
|
||||||
#
|
#
|
||||||
class MikesBackup:
|
class MikesBackup:
|
||||||
|
|
||||||
#
|
#
|
||||||
_log_dir = None
|
__log_dir = None
|
||||||
_remote_host = None
|
|
||||||
_remote_user = None
|
|
||||||
_destination_dir_base = None
|
|
||||||
#
|
|
||||||
_ssh_key = None
|
|
||||||
_quiet_ssh = True
|
|
||||||
#
|
|
||||||
_source_dirs = []
|
|
||||||
_source_dir_excludes = []
|
|
||||||
#
|
|
||||||
_force_full = False
|
|
||||||
_force_differential = False
|
|
||||||
|
|
||||||
|
__remote_host = None
|
||||||
|
__remote_user = None
|
||||||
|
__destination_dir_base = None
|
||||||
|
|
||||||
|
__source_dir = None
|
||||||
|
__source_dir_includes = []
|
||||||
|
__source_dir_excludes = []
|
||||||
|
__source_mountpoint_demands = []
|
||||||
|
|
||||||
|
__ssh_key = None
|
||||||
|
__quiet_ssh = True
|
||||||
|
|
||||||
|
__force_full = False
|
||||||
|
__force_differential = False
|
||||||
|
__no_incremental = False
|
||||||
|
|
||||||
|
__log_file_handle = None
|
||||||
|
|
||||||
|
CONST_FULL_DIRECTORY_NAME = "full"
|
||||||
|
CONST_DIFFERENTIAL_DIRECTORY_NAME = "differential"
|
||||||
|
CONST_DIFFERENTIAL_IN_PROGRESS_DIRECTORY_NAME = "IN-PROGRESS"
|
||||||
|
|
||||||
#
|
#
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.ParseArgs()
|
|
||||||
|
self.parse_args()
|
||||||
|
|
||||||
|
#
|
||||||
|
def __del__(self):
|
||||||
|
|
||||||
|
self.close_log_file()
|
||||||
|
|
||||||
|
#
|
||||||
|
def __str__(self):
|
||||||
|
|
||||||
|
s = ""
|
||||||
|
|
||||||
|
s += "MikesBackup Class Instance"
|
||||||
|
s += "\nLog Dir: " + str(self.__log_dir)
|
||||||
|
s += "\nRemote Host: " + str(self.__remote_host)
|
||||||
|
s += "\nRemote User: " + str(self.__remote_user)
|
||||||
|
s += "\nDestination Dir Base: " + str(self.__destination_dir_base)
|
||||||
|
s += "\nSource Dir (Main): " + str(self.__source_dir)
|
||||||
|
s += "\nSource Dirs (Includes): " + str(self.__source_dir_includes)
|
||||||
|
s += "\nSource Dirs (Excludes): " + str(self.__source_dir_excludes)
|
||||||
|
s += "\nSource Mountpoint Demands: " + str(self.__source_mountpoint_demands)
|
||||||
|
s += "\nSSH Key: " + str(self.__ssh_key)
|
||||||
|
s += "\nQuiet SSH: " + str(self.__quiet_ssh)
|
||||||
|
s += "\nForce Full Backup: " + str(self.__force_full)
|
||||||
|
s += "\nForce Differential: " + str(self.__force_differential)
|
||||||
|
s += "\nDisallow Incremental: " + str(self.__no_incremental)
|
||||||
|
|
||||||
|
return s
|
||||||
|
|
||||||
|
#
|
||||||
|
def log(self, s, o=None):
|
||||||
|
|
||||||
|
the_date = self.get_datetime_for_logging()
|
||||||
|
|
||||||
|
to_log = "[MikesBackup][" + the_date + "] " + s
|
||||||
|
|
||||||
|
# Print the log line
|
||||||
|
print(to_log)
|
||||||
|
|
||||||
|
# Append the log line to the log file
|
||||||
|
f = self.open_log_file()
|
||||||
|
if f:
|
||||||
|
f.write(to_log + "\n")
|
||||||
|
|
||||||
|
# Recurse in order to print whatever o is outputting, if anything
|
||||||
|
if o is not None:
|
||||||
|
o_lines = str(o).split("\n")
|
||||||
|
for line in o_lines:
|
||||||
|
self.log(line)
|
||||||
|
|
||||||
|
#
|
||||||
|
def open_log_file(self):
|
||||||
|
|
||||||
|
if self.__log_file_handle:
|
||||||
|
return self.__log_file_handle
|
||||||
|
|
||||||
|
log_file_path = self.make_log_path()
|
||||||
|
if log_file_path:
|
||||||
|
self.__log_file_handle = open(log_file_path, "w")
|
||||||
|
|
||||||
|
return self.__log_file_handle
|
||||||
|
|
||||||
|
#
|
||||||
|
def close_log_file(self):
|
||||||
|
|
||||||
|
if self.__log_file_handle:
|
||||||
|
self.__log_file_handle.close()
|
||||||
|
self.__log_file_handle = None
|
||||||
|
|
||||||
#
|
#
|
||||||
def eprint(*args, **kwargs):
|
def eprint(*args, **kwargs):
|
||||||
|
|
||||||
print(*args, file=sys.stderr, **kwargs)
|
print(*args, file=sys.stderr, **kwargs)
|
||||||
|
|
||||||
#
|
#
|
||||||
def log():
|
def parse_args(self):
|
||||||
pass
|
|
||||||
|
|
||||||
#
|
|
||||||
def ParseArgs(self):
|
|
||||||
|
|
||||||
# Simple mapped args
|
|
||||||
args_map = {
|
|
||||||
"--log-dir" : "_log_dir",
|
|
||||||
"--remote-host" : "_remote_host",
|
|
||||||
"--remote-user" : "_remote_user",
|
|
||||||
"--destination-dir" : "_destination_dir_base",
|
|
||||||
"--ssh-key" : "_ssh_key"
|
|
||||||
}
|
|
||||||
|
|
||||||
#
|
#
|
||||||
print()
|
print()
|
||||||
print ("Parsing arguments")
|
self.log("Parsing arguments")
|
||||||
a = 1
|
a = 0
|
||||||
while a < len(sys.argv):
|
while a + 1 < len(sys.argv):
|
||||||
|
|
||||||
|
a += 1
|
||||||
|
|
||||||
#
|
#
|
||||||
arg = sys.argv[a]
|
arg = sys.argv[a]
|
||||||
|
|
||||||
#
|
#
|
||||||
valid_arg = False
|
valid_arg = False
|
||||||
for arg_name in args_map:
|
|
||||||
if arg == arg_name:
|
|
||||||
valid_arg = True
|
|
||||||
self_var_name = args_map[arg_name]
|
|
||||||
self_var_value = sys.argv[ a + 1 ]
|
|
||||||
self.__dict__[self_var_name] = self_var_value
|
|
||||||
print("Found argument \"", arg_name, "\" ==>", self_var_value)
|
|
||||||
a = a + 1
|
|
||||||
break
|
|
||||||
if arg == "":
|
if arg == "":
|
||||||
valid_arg = True
|
valid_arg = True
|
||||||
elif arg == "--full":
|
elif arg == "--full":
|
||||||
valid_arg = True
|
valid_arg = True
|
||||||
self._force_full = True
|
self.__force_full = True
|
||||||
|
self.log("Forcing a full backup")
|
||||||
elif arg == "--diff" or arg == "--differential":
|
elif arg == "--diff" or arg == "--differential":
|
||||||
valid_arg = True
|
valid_arg = True
|
||||||
self._force_differential = True
|
self.__force_differential = True
|
||||||
|
self.log("Forcing a differential backup")
|
||||||
|
elif arg == "--no-incremental":
|
||||||
|
valid_arg = True
|
||||||
|
self.__no_incremental = True
|
||||||
|
self.log("Disallowing incremental backups (differentials will only link back to full)")
|
||||||
|
elif arg == "--log-dir":
|
||||||
|
valid_arg = True
|
||||||
|
self.__log_dir = sys.argv[a + 1]
|
||||||
|
self.log("Found log dir: " + self.__log_dir)
|
||||||
|
a = a + 1
|
||||||
elif arg == "--source-dir":
|
elif arg == "--source-dir":
|
||||||
valid_arg = True
|
valid_arg = True
|
||||||
self._source_dirs.append( sys.argv[ a + 1 ] )
|
if self.__source_dir:
|
||||||
print("Found source dir:", sys.argv[ a + 1 ])
|
raise Exception("--source-dir can only be used once")
|
||||||
|
self.__source_dir = sys.argv[a + 1]
|
||||||
|
self.log("Found source dir: " + sys.argv[a + 1])
|
||||||
|
a = a + 1
|
||||||
|
elif arg == "--include":
|
||||||
|
valid_arg = True
|
||||||
|
self.__source_dir_includes.append(sys.argv[a + 1])
|
||||||
|
self.log("Found additional source dir include: " + sys.argv[a + 1])
|
||||||
|
a = a + 1
|
||||||
|
elif arg == "--source-mountpoint":
|
||||||
|
valid_arg = True
|
||||||
|
self.__source_mountpoint_demands.append(sys.argv[a + 1])
|
||||||
|
self.log("Found required source mountpoint: " + sys.argv[a + 1])
|
||||||
|
a += 1
|
||||||
|
elif arg == "--destination-dir":
|
||||||
|
valid_arg = True
|
||||||
|
self.__destination_dir_base = sys.argv[a + 1]
|
||||||
|
self.log("Found destination dir: " + self.__destination_dir_base)
|
||||||
a = a + 1
|
a = a + 1
|
||||||
elif arg == "--exclude":
|
elif arg == "--exclude":
|
||||||
valid_arg = True
|
valid_arg = True
|
||||||
self._source_dir_excludes.append( sys.argv[ a + 1 ] )
|
self.__source_dir_excludes.append(sys.argv[a + 1])
|
||||||
print("Found exclude dir:", sys.argv[ a + 1 ])
|
self.log("Found exclude dir: " + sys.argv[a + 1])
|
||||||
a = a + 1
|
a = a + 1
|
||||||
|
elif arg == "--remote-host":
|
||||||
|
valid_arg = True
|
||||||
|
self.__remote_host = sys.argv[a + 1]
|
||||||
|
self.log("Found remote host: " + self.__remote_host)
|
||||||
|
a = a + 1
|
||||||
|
elif arg == "--remote-user":
|
||||||
|
valid_arg = True
|
||||||
|
self.__remote_user = sys.argv[a + 1]
|
||||||
|
self.log("Found remote user: " + self.__remote_user)
|
||||||
|
a = a + 1
|
||||||
|
elif arg == "--ssh-key":
|
||||||
|
valid_arg = True
|
||||||
|
self.__ssh_key = sys.argv[a + 1]
|
||||||
|
self.log("Found ssh key: " + self.__ssh_key)
|
||||||
a = a + 1
|
a = a + 1
|
||||||
|
|
||||||
#
|
#
|
||||||
if not valid_arg:
|
if not valid_arg:
|
||||||
raise Exception("Invalid argument:", arg);
|
raise Exception("Invalid argument:", arg)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_datetime_for_logging():
|
||||||
|
|
||||||
#
|
#
|
||||||
def GetDatetimeForFilename(self):
|
return datetime.datetime.now().strftime("%b %d %Y; %I%M%p")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_datetime_for_filename():
|
||||||
|
|
||||||
#
|
#
|
||||||
return datetime.datetime.now().strftime('%Y-%b-%d_%I%M%p')
|
return datetime.datetime.now().strftime('%Y-%b-%d_%I%M%p')
|
||||||
|
|
||||||
#
|
#
|
||||||
def IsUsingSSH(self):
|
def is_using_source_mountpoints(self):
|
||||||
|
|
||||||
|
return len(self.__source_mountpoint_demands) > 0
|
||||||
|
|
||||||
#
|
#
|
||||||
if self._remote_host != None or self._remote_user != None or self._ssh_key != None:
|
def demand_source_mountpoints(self):
|
||||||
|
|
||||||
|
for mountpoint_path in self.__source_mountpoint_demands:
|
||||||
|
if not os.path.ismount(mountpoint_path):
|
||||||
|
raise Exception("Required mountpoint is not mounted: " + str(mountpoint_path))
|
||||||
|
self.log("Verified mountpoint: " + mountpoint_path)
|
||||||
|
|
||||||
|
#
|
||||||
|
def is_using_ssh(self):
|
||||||
|
|
||||||
|
#
|
||||||
|
if (
|
||||||
|
self.__remote_host is not None
|
||||||
|
or self.__remote_user is not None
|
||||||
|
or self.__ssh_key is not None
|
||||||
|
):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
#
|
#
|
||||||
def DemandSSHConfig(self):
|
def demand_ssh_config(self):
|
||||||
|
|
||||||
#
|
#
|
||||||
if self.IsUsingSSH():
|
if self.is_using_ssh():
|
||||||
if self._remote_host == None:
|
if self.__remote_host is None:
|
||||||
raise Exception("Please provide remote host")
|
raise Exception("Please provide remote host")
|
||||||
if self._remote_user == None:
|
if self.__remote_user is None:
|
||||||
raise Exception("Please provide remote user")
|
raise Exception("Please provide remote user")
|
||||||
|
|
||||||
#
|
#
|
||||||
def DemandDestinationDirectoryConfig(self):
|
def demand_source_directory_config(self):
|
||||||
|
|
||||||
#
|
#
|
||||||
if self._destination_dir_base == None:
|
if self.__source_dir is None:
|
||||||
|
raise Exception("Please provide a source directory")
|
||||||
|
|
||||||
|
#
|
||||||
|
def demand_destination_directory_config(self):
|
||||||
|
|
||||||
|
#
|
||||||
|
if self.__destination_dir_base is None:
|
||||||
raise Exception("Please provide backup destination directory")
|
raise Exception("Please provide backup destination directory")
|
||||||
|
|
||||||
#
|
#
|
||||||
def DoesDestinationDirectoryExist(self, destination_path):
|
def does_destination_directory_exist(self, destination_path):
|
||||||
|
|
||||||
#
|
#
|
||||||
print("Trying to determine if destination path exists:", destination_path)
|
self.log("Trying to determine if destination path exists:" + destination_path)
|
||||||
|
|
||||||
# Local?
|
# Local?
|
||||||
if not self.IsUsingSSH():
|
if not self.is_using_ssh():
|
||||||
print("Checking for local destination path")
|
self.log("Checking for local destination path")
|
||||||
if os.path.isdir(destination_path):
|
if os.path.isdir(destination_path):
|
||||||
print("Local destination path exists")
|
self.log("Local destination path exists")
|
||||||
return True
|
return True
|
||||||
else:
|
else:
|
||||||
print("Local destination path does not exist")
|
self.log("Local destination path does not exist")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
#
|
#
|
||||||
print("Checking for remote destination path")
|
self.log("Checking for remote destination path")
|
||||||
command = [
|
command = [
|
||||||
"[ -d " + destination_path + " ]"
|
"[ -d " + destination_path + " ]"
|
||||||
]
|
]
|
||||||
|
|
||||||
#
|
#
|
||||||
code, stdout, stderr = self.ExecuteRemoteSSHCommand(command)
|
code, stdout, stderr = self.execute_remote_ssh_command(command)
|
||||||
if code == 0:
|
if code == 0:
|
||||||
print("Remote destination dir was found: " + destination_path)
|
self.log("Remote destination dir was found: " + destination_path)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
#
|
#
|
||||||
print("Remote dir didn't seem to exist: " + destination_path)
|
self.log("Remote dir didn't seem to exist: " + destination_path)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
#
|
#
|
||||||
def DemandDestinationBaseBackupDirectory(self):
|
def demand_destination_base_backup_directory(self):
|
||||||
|
|
||||||
#
|
#
|
||||||
self.DemandDestinationDirectoryConfig()
|
self.demand_destination_directory_config()
|
||||||
|
|
||||||
#
|
#
|
||||||
destination_path = self._destination_dir_base
|
destination_path = self.__destination_dir_base
|
||||||
|
|
||||||
#
|
#
|
||||||
if self.DoesDestinationDirectoryExist(destination_path) == False:
|
if self.does_destination_directory_exist(destination_path) is False:
|
||||||
raise Exception("Backup destination directory doesn't exist: " + destination_path)
|
raise Exception("Backup destination directory doesn't exist: " + destination_path)
|
||||||
|
|
||||||
#
|
#
|
||||||
def DoesFullBackupDestinationDirectoryExist(self):
|
def does_full_backup_destination_directory_exist(self):
|
||||||
|
|
||||||
#
|
#
|
||||||
dir_path = self.MakeFullBackupDestinationPath()
|
dir_path = self.make_full_backup_destination_path()
|
||||||
|
|
||||||
#
|
#
|
||||||
print("Trying to determine if Full backup destination directory exists:", dir_path)
|
self.log("Trying to determine if Full backup destination directory exists:", dir_path)
|
||||||
return self.DoesDestinationDirectoryExist(dir_path)
|
return self.does_destination_directory_exist(dir_path)
|
||||||
|
|
||||||
#
|
#
|
||||||
def GetSourceDirectories(self):
|
def do_backup(self):
|
||||||
|
|
||||||
#
|
|
||||||
if len(self._source_dirs) == 0:
|
|
||||||
raise Exception("No source directories specified")
|
|
||||||
|
|
||||||
return self._source_dirs
|
|
||||||
|
|
||||||
#
|
|
||||||
def DoBackup(self):
|
|
||||||
|
|
||||||
#
|
#
|
||||||
print()
|
print()
|
||||||
print("Enter: DoBackup")
|
self.log("Enter: do_backup")
|
||||||
|
|
||||||
|
# Source mountpoints must be mounted
|
||||||
|
self.demand_source_mountpoints()
|
||||||
|
|
||||||
# Remote base dir must exist
|
# Remote base dir must exist
|
||||||
self.DemandDestinationBaseBackupDirectory()
|
self.demand_destination_base_backup_directory()
|
||||||
|
|
||||||
# Forced full or differential by args?
|
# Forced full or differential by args?
|
||||||
if self._force_full == True or self._force_differential == True:
|
if self.__force_full is True or self.__force_differential is True:
|
||||||
if self._force_full == True:
|
if self.__force_full is True:
|
||||||
print("Forcing full backup")
|
self.log("Forcing full backup")
|
||||||
self.DoFullBackup()
|
self.do_full_backup()
|
||||||
else:
|
else:
|
||||||
print("Forcing differential backup")
|
self.log("Forcing differential backup")
|
||||||
self.DoDifferentialBackup()
|
self.do_differential_backup()
|
||||||
return
|
return
|
||||||
|
|
||||||
# Automatically choose full or differential
|
# Automatically choose full or differential
|
||||||
if self.DoesFullBackupDestinationDirectoryExist():
|
if self.does_full_backup_destination_directory_exist():
|
||||||
print("Automatically choosing differential backup, because full backup destination directory already exists")
|
self.log("Automatically choosing differential backup, because full backup destination directory already exists")
|
||||||
self.DoDifferentialBackup()
|
self.do_differential_backup()
|
||||||
else:
|
else:
|
||||||
print("Automatically choosing full backup, because full backup destination directory wasn't found")
|
self.log("Automatically choosing full backup, because full backup destination directory wasn't found")
|
||||||
self.DoFullBackup()
|
self.do_full_backup()
|
||||||
|
|
||||||
#
|
#
|
||||||
def DoFullBackup(self):
|
def do_full_backup(self):
|
||||||
|
|
||||||
# Start args
|
# Start args
|
||||||
args = []
|
args = []
|
||||||
|
|
||||||
# Get destination directory
|
# Get destination directory
|
||||||
destination_dir = self.MakeFullBackupDestinationPath()
|
destination_dir = self.make_full_backup_destination_path()
|
||||||
|
|
||||||
# Append source directories
|
# Append source directory
|
||||||
args.extend(self.GetSourceDirectories())
|
args.append(self.make_rsync_source_directory_part())
|
||||||
|
|
||||||
# Append remote destination directory
|
# Append remote destination directory
|
||||||
#args.append( self._remote_user + "@" + self._remote_host + ":" + remote_dir)
|
# args.append( self.__remote_user + "@" + self.__remote_host + ":" + remote_dir)
|
||||||
args.append( self.MakeRsyncRemoteDestinationPart(destination_dir) )
|
args.append(self.make_rsync_remote_destination_part(destination_dir))
|
||||||
|
|
||||||
# print("Args", str(args))
|
# print("Args", str(args))
|
||||||
print("Destination dir:", destination_dir)
|
self.log("Destination dir:" + destination_dir)
|
||||||
|
|
||||||
self.ExecuteRsync(args)
|
self.execute_rsync(args)
|
||||||
|
|
||||||
|
self.log("Rsync seems to have finished successfully")
|
||||||
|
self.log("Because a full backup has succeeded, will now delete any differential backups")
|
||||||
|
self.execute_remote_ssh_command(
|
||||||
|
[
|
||||||
|
"rm",
|
||||||
|
"-rfv",
|
||||||
|
self.make_remote_differential_backup_path_base()
|
||||||
|
])
|
||||||
|
self.log("Finished deleting old differentials")
|
||||||
|
|
||||||
#
|
#
|
||||||
def DoDifferentialBackup(self):
|
def do_differential_backup(self):
|
||||||
|
|
||||||
# Start args
|
# Start args
|
||||||
args = []
|
args = []
|
||||||
|
|
||||||
# Get directories
|
# Get directories
|
||||||
link_dest_dir = self.MakeFullBackupDestinationPath()
|
link_dest_dir = self.determine_rsync_backup_link_destination_path()
|
||||||
destination_dir = self.MakeRemoteDifferentialBackupPath()
|
destination_dir_in_progress = self.make_remote_differential_in_progress_backup_path()
|
||||||
self.EnsureDestinationDirectory(destination_dir)
|
destination_dir_final = self.make_remote_differential_backup_path()
|
||||||
|
self.ensure_destination_directory(destination_dir_in_progress)
|
||||||
|
|
||||||
# Add link dest arg
|
# Add link dest arg?
|
||||||
|
if link_dest_dir:
|
||||||
args.append("--link-dest")
|
args.append("--link-dest")
|
||||||
args.append(link_dest_dir)
|
args.append(link_dest_dir)
|
||||||
|
|
||||||
# Append source directories
|
# Append source directory
|
||||||
args.extend(self.GetSourceDirectories())
|
args.append(self.make_rsync_source_directory_part())
|
||||||
|
|
||||||
# Append remote destination directory
|
# Append remote destination directory
|
||||||
#args.append( self._remote_user + "@" + self._remote_host + ":" + remote_dir)
|
args.append(self.make_rsync_remote_destination_part(destination_dir_in_progress))
|
||||||
args.append( self.MakeRsyncRemoteDestinationPart(destination_dir) )
|
|
||||||
|
|
||||||
#print("Args", str(args))
|
self.log("Link destination dir: " + link_dest_dir)
|
||||||
print("Link destination dir:", link_dest_dir)
|
self.log("Destination dir: " + destination_dir_in_progress)
|
||||||
print("Destination dir:", destination_dir)
|
|
||||||
|
|
||||||
self.ExecuteRsync(args)
|
self.execute_rsync(args)
|
||||||
|
|
||||||
|
self.log("Rsync seems to have finished successfully")
|
||||||
|
self.log("Renaming temporary directory")
|
||||||
|
self.log("Old: " + destination_dir_in_progress)
|
||||||
|
self.log("New: " + destination_dir_final)
|
||||||
|
if self.is_using_ssh():
|
||||||
|
return_code, stdout, stderr = self.execute_remote_ssh_command([
|
||||||
|
"mv",
|
||||||
|
destination_dir_in_progress,
|
||||||
|
destination_dir_final
|
||||||
|
])
|
||||||
|
if return_code != 0:
|
||||||
|
raise Exception("Failed to move temporary diff directory to its final home")
|
||||||
|
else:
|
||||||
|
os.rename(destination_dir_in_progress, destination_dir_final)
|
||||||
|
self.log("Rename was successful")
|
||||||
|
|
||||||
#
|
#
|
||||||
def MakeLogDirectoryPath(self):
|
def make_log_directory_path(self):
|
||||||
|
|
||||||
#
|
#
|
||||||
log_dir = self._log_dir
|
log_dir = self.__log_dir
|
||||||
if log_dir == None:
|
if log_dir is None:
|
||||||
print("No log directory specified; Defaulting to current working directory")
|
print("No log directory specified; Won't log")
|
||||||
log_dir = "."
|
return None
|
||||||
|
|
||||||
return log_dir
|
return log_dir
|
||||||
|
|
||||||
#
|
#
|
||||||
def MakeLogPath(self):
|
def make_log_path(self):
|
||||||
|
|
||||||
# Log dir
|
# Log dir
|
||||||
log_dir = self.MakeLogDirectoryPath()
|
log_dir = self.make_log_directory_path()
|
||||||
|
if not log_dir:
|
||||||
|
return None
|
||||||
|
|
||||||
# Path
|
# Path
|
||||||
log_path = os.path.join(log_dir, self.GetDatetimeForFilename() + ".log")
|
log_path = os.path.join(log_dir, self.get_datetime_for_filename() + ".log")
|
||||||
|
|
||||||
return log_path
|
return log_path
|
||||||
|
|
||||||
#
|
#
|
||||||
def MakeFullBackupDestinationPath(self):
|
def make_full_backup_destination_path(self):
|
||||||
|
|
||||||
#
|
#
|
||||||
if self._destination_dir_base == None:
|
if self.__destination_dir_base is None:
|
||||||
raise Exception("No remote directory was specified")
|
raise Exception("No remote directory was specified")
|
||||||
|
|
||||||
#
|
#
|
||||||
return os.path.join(self._destination_dir_base, "Full")
|
return os.path.join(self.__destination_dir_base, self.CONST_FULL_DIRECTORY_NAME)
|
||||||
|
|
||||||
#
|
#
|
||||||
def MakeRemoteDifferentialBackupPath(self):
|
def make_remote_differential_backup_path_base(self):
|
||||||
|
|
||||||
#
|
#
|
||||||
if self._destination_dir_base == None:
|
if self.__destination_dir_base is None:
|
||||||
raise Exception("No remote directory was specified")
|
raise Exception("No remote directory was specified")
|
||||||
|
|
||||||
#
|
return os.path.join(self.__destination_dir_base, self.CONST_DIFFERENTIAL_DIRECTORY_NAME)
|
||||||
return os.path.join(self._destination_dir_base, "Differential", self.GetDatetimeForFilename())
|
|
||||||
|
|
||||||
#
|
#
|
||||||
def MakeRsyncRemoteDestinationPart(self, destination_dir):
|
def make_remote_differential_in_progress_backup_path(self):
|
||||||
|
|
||||||
|
diff_path_base = self.make_remote_differential_backup_path_base()
|
||||||
|
|
||||||
|
return os.path.join(diff_path_base, self.CONST_DIFFERENTIAL_IN_PROGRESS_DIRECTORY_NAME)
|
||||||
|
|
||||||
|
#
|
||||||
|
def make_remote_differential_backup_path(self):
|
||||||
|
|
||||||
|
diff_path_base = self.make_remote_differential_backup_path_base()
|
||||||
|
|
||||||
|
#
|
||||||
|
return os.path.join(diff_path_base, self.get_datetime_for_filename())
|
||||||
|
|
||||||
|
#
|
||||||
|
def make_rsync_source_includes_part(self):
|
||||||
|
|
||||||
|
args = []
|
||||||
|
|
||||||
|
for d in self.__source_dir_includes:
|
||||||
|
args.append("--include")
|
||||||
|
args.append(d)
|
||||||
|
|
||||||
|
return args
|
||||||
|
|
||||||
|
#
|
||||||
|
def make_rsync_source_directory_part(self):
|
||||||
|
|
||||||
|
self.demand_source_directory_config()
|
||||||
|
|
||||||
|
#
|
||||||
|
return self.__source_dir
|
||||||
|
|
||||||
|
#
|
||||||
|
def make_rsync_remote_destination_part(self, destination_dir):
|
||||||
|
|
||||||
#
|
#
|
||||||
part = ""
|
part = ""
|
||||||
|
|
||||||
#
|
#
|
||||||
if self._remote_host != None:
|
if self.__remote_host is not None:
|
||||||
if self._remote_user != None:
|
if self.__remote_user is not None:
|
||||||
part += self._remote_user + "@"
|
part += self.__remote_user + "@"
|
||||||
part += self._remote_host + ":"
|
part += self.__remote_host + ":"
|
||||||
|
|
||||||
#
|
#
|
||||||
part += destination_dir
|
part += destination_dir
|
||||||
@ -328,53 +515,157 @@ class MikesBackup:
|
|||||||
return part
|
return part
|
||||||
|
|
||||||
#
|
#
|
||||||
def EnsureLocalDirectory(self, d):
|
def determine_rsync_backup_link_destination_path(self):
|
||||||
|
|
||||||
|
self.demand_destination_directory_config()
|
||||||
|
|
||||||
|
self.log("Begin trying to determine which previous backup path to use as link")
|
||||||
|
|
||||||
|
#
|
||||||
|
newest_path = None
|
||||||
|
newest_path_date = None
|
||||||
|
|
||||||
|
# Pattern to parse the 'ls' command
|
||||||
|
pattern = re.compile(
|
||||||
|
""".*(?P<date>[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9} .{5}) (?P<name>.+)$""",
|
||||||
|
re.MULTILINE
|
||||||
|
)
|
||||||
|
|
||||||
|
# Get listing info for the full path
|
||||||
|
destination_path_full = self.make_full_backup_destination_path()
|
||||||
|
args_full_destination_path_ls = [
|
||||||
|
"ls",
|
||||||
|
"-l",
|
||||||
|
"-c",
|
||||||
|
"--all",
|
||||||
|
"--full-time",
|
||||||
|
destination_path_full
|
||||||
|
]
|
||||||
|
if self.is_using_ssh():
|
||||||
|
return_code, stdout, stderr = self.execute_remote_ssh_command(args_full_destination_path_ls)
|
||||||
|
else:
|
||||||
|
return_code, stdout, stderr = self.execute_command(args_full_destination_path_ls)
|
||||||
|
if return_code != 0:
|
||||||
|
raise Exception("Failed to get listing info for base destination directory")
|
||||||
|
for match in pattern.finditer(stdout):
|
||||||
|
name = match.group("name")
|
||||||
|
date = match.group("date")
|
||||||
|
if name == ".":
|
||||||
|
self.log("Start by assuming \"Full\" is the most recent backup: " + destination_path_full)
|
||||||
|
self.log("; With a date of " + date)
|
||||||
|
newest_path = destination_path_full
|
||||||
|
newest_path_date = date
|
||||||
|
break
|
||||||
|
if not newest_path:
|
||||||
|
self.log("Didn't find a \"Full\" backup on remote")
|
||||||
|
|
||||||
|
if self.__no_incremental:
|
||||||
|
self.log("Incremental backups are disabled; Won't consider any differential directories for the link target")
|
||||||
|
return newest_path
|
||||||
|
|
||||||
|
# Get listing info for all differential directories
|
||||||
|
differential_path_base = self.make_remote_differential_backup_path_base()
|
||||||
|
self.ensure_destination_directory(differential_path_base)
|
||||||
|
args_differential_destination_path_ls = [
|
||||||
|
"ls",
|
||||||
|
"-l",
|
||||||
|
"-c",
|
||||||
|
"--all",
|
||||||
|
"--full-time",
|
||||||
|
differential_path_base
|
||||||
|
]
|
||||||
|
if self.is_using_ssh():
|
||||||
|
return_code, stdout, stderr = self.execute_remote_ssh_command(args_differential_destination_path_ls)
|
||||||
|
else:
|
||||||
|
return_code, stdout, stderr = self.execute_command(args_differential_destination_path_ls)
|
||||||
|
if return_code != 0:
|
||||||
|
raise Exception("Failed to get listing info for destination differential base directory")
|
||||||
|
|
||||||
|
# Look for the most recent differential directory
|
||||||
|
# (must be newer than the Full directory too)
|
||||||
|
for match in pattern.finditer(stdout):
|
||||||
|
|
||||||
|
name = match.group("name")
|
||||||
|
date = match.group("date")
|
||||||
|
|
||||||
|
if name == "." or name == ".." or name == self.CONST_DIFFERENTIAL_IN_PROGRESS_DIRECTORY_NAME:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if newest_path is None or date > newest_path_date:
|
||||||
|
self.log("Found a newer differential backup: " + name + "; " + date)
|
||||||
|
newest_path = os.path.join(differential_path_base, name)
|
||||||
|
newest_path_date = date
|
||||||
|
else:
|
||||||
|
self.log("Not newer: " + name + "; " + date)
|
||||||
|
|
||||||
|
#
|
||||||
|
self.log("Newest backup path is: " + newest_path)
|
||||||
|
self.log("; With a date of: " + newest_path_date)
|
||||||
|
|
||||||
|
return newest_path
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def ensure_local_directory(d):
|
||||||
|
|
||||||
#
|
#
|
||||||
if not os.path.exists(d):
|
if not os.path.exists(d):
|
||||||
os.makedirs(d)
|
os.makedirs(d)
|
||||||
|
|
||||||
#
|
#
|
||||||
def EnsureDestinationDirectory(self, d):
|
def ensure_destination_directory(self, d):
|
||||||
|
|
||||||
#
|
#
|
||||||
if not self.DoesDestinationDirectoryExist(d):
|
if not self.does_destination_directory_exist(d):
|
||||||
|
|
||||||
#
|
#
|
||||||
print("Destination directory doesn't exist; Will create:", d)
|
self.log("Destination directory doesn't exist; Will create:" + d)
|
||||||
|
|
||||||
#
|
#
|
||||||
if self.IsUsingSSH():
|
if self.is_using_ssh():
|
||||||
command = [
|
command = [
|
||||||
"mkdir",
|
"mkdir",
|
||||||
"--parents",
|
"--parents",
|
||||||
d
|
d
|
||||||
]
|
]
|
||||||
self.ExecuteRemoteSSHCommand(command)
|
self.execute_remote_ssh_command(command)
|
||||||
else:
|
else:
|
||||||
os.makedirs(d)
|
os.makedirs(d)
|
||||||
|
|
||||||
#
|
#
|
||||||
def StartRsyncArgs(self):
|
def start_rsync_args(self):
|
||||||
|
|
||||||
#
|
#
|
||||||
self.EnsureLocalDirectory(self.MakeLogDirectoryPath())
|
|
||||||
args = [
|
args = [
|
||||||
"rsync",
|
"rsync",
|
||||||
"--log-file", self.MakeLogPath(),
|
|
||||||
"--archive",
|
"--archive",
|
||||||
"--compress",
|
"--compress",
|
||||||
"--progress",
|
"--progress",
|
||||||
"--stats",
|
"--stats",
|
||||||
"--human-readable",
|
"--human-readable",
|
||||||
"--itemize-changes",
|
"--itemize-changes",
|
||||||
"--one-file-system",
|
|
||||||
"--delete",
|
"--delete",
|
||||||
"--delete-excluded"
|
"--delete-excluded"
|
||||||
]
|
]
|
||||||
|
|
||||||
|
log_dir = self.make_log_directory_path()
|
||||||
|
log_path = self.make_log_path()
|
||||||
|
if log_dir and log_path:
|
||||||
|
self.ensure_local_directory(log_dir)
|
||||||
|
args.append("--log-file")
|
||||||
|
args.append(log_path)
|
||||||
|
|
||||||
|
# Only allow recursion into multiple file systems
|
||||||
|
# if any mountpoints were specified
|
||||||
|
if not self.is_using_source_mountpoints():
|
||||||
|
args.append("--one-file-system")
|
||||||
|
|
||||||
#
|
#
|
||||||
for e in self._source_dir_excludes:
|
for i in self.__source_dir_includes:
|
||||||
|
args.append("--include")
|
||||||
|
args.append(i)
|
||||||
|
|
||||||
|
#
|
||||||
|
for e in self.__source_dir_excludes:
|
||||||
args.append("--exclude")
|
args.append("--exclude")
|
||||||
args.append(e)
|
args.append(e)
|
||||||
|
|
||||||
@ -384,44 +675,67 @@ class MikesBackup:
|
|||||||
return args
|
return args
|
||||||
|
|
||||||
#
|
#
|
||||||
def StartRsyncEnvironmentalVariables(self):
|
def start_rsync_environment_variables(self):
|
||||||
|
|
||||||
#
|
#
|
||||||
env = {}
|
env = {}
|
||||||
|
|
||||||
#
|
#
|
||||||
if self._ssh_key != None or self._quiet_ssh == True:
|
if self.__ssh_key is not None or self.__quiet_ssh is True:
|
||||||
env["RSYNC_RSH"] = "ssh"
|
env["RSYNC_RSH"] = "ssh"
|
||||||
if self._ssh_key != None:
|
if self.__ssh_key is not None:
|
||||||
env["RSYNC_RSH"] += " -i " + shlex.quote(self._ssh_key)
|
env["RSYNC_RSH"] += " -i " + shlex.quote(self.__ssh_key)
|
||||||
if self._quiet_ssh == True:
|
if self.__quiet_ssh is True:
|
||||||
env["RSYNC_RSH"] += " -q"
|
env["RSYNC_RSH"] += " -q"
|
||||||
|
|
||||||
return env
|
return env
|
||||||
|
|
||||||
#
|
@staticmethod
|
||||||
def ExecuteRemoteSSHCommand(self, command):
|
def execute_command(command):
|
||||||
|
|
||||||
#
|
#
|
||||||
self.DemandSSHConfig()
|
args = list()
|
||||||
|
|
||||||
|
# Append the command
|
||||||
|
if isinstance(command, str):
|
||||||
|
args.append(command)
|
||||||
|
elif isinstance(command, list):
|
||||||
|
args.extend(command)
|
||||||
|
else:
|
||||||
|
raise Exception("Unsupported command datatype")
|
||||||
|
|
||||||
|
# Spawn
|
||||||
|
process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
stdout, stderr = process.communicate()
|
||||||
|
|
||||||
|
stdout = stdout.decode()
|
||||||
|
stderr = stderr.decode()
|
||||||
|
|
||||||
|
return process.returncode, stdout, stderr
|
||||||
|
|
||||||
#
|
#
|
||||||
args = []
|
def execute_remote_ssh_command(self, command):
|
||||||
|
|
||||||
|
#
|
||||||
|
self.demand_ssh_config()
|
||||||
|
|
||||||
|
#
|
||||||
|
args = list()
|
||||||
|
|
||||||
# ssh command
|
# ssh command
|
||||||
args.append("ssh")
|
args.append("ssh")
|
||||||
|
|
||||||
# Quiet?
|
# Quiet?
|
||||||
if self._quiet_ssh == True:
|
if self.__quiet_ssh is True:
|
||||||
args.append("-q")
|
args.append("-q")
|
||||||
|
|
||||||
# ssh key
|
# ssh key
|
||||||
if self._ssh_key != None:
|
if self.__ssh_key is not None:
|
||||||
args.append("-i")
|
args.append("-i")
|
||||||
args.append(self._ssh_key)
|
args.append(self.__ssh_key)
|
||||||
|
|
||||||
# ssh user@host
|
# ssh user@host
|
||||||
args.append(self._remote_user + "@" + self._remote_host)
|
args.append(self.__remote_user + "@" + self.__remote_host)
|
||||||
|
|
||||||
# Append the command
|
# Append the command
|
||||||
args.append("--")
|
args.append("--")
|
||||||
@ -432,32 +746,41 @@ class MikesBackup:
|
|||||||
else:
|
else:
|
||||||
raise Exception("Unsupported command datatype")
|
raise Exception("Unsupported command datatype")
|
||||||
|
|
||||||
# Spawn SSH in shell
|
# Spawn
|
||||||
#process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
process = subprocess.Popen(args)
|
# process = subprocess.Popen(args)
|
||||||
#stdout, stderr = process.communicate()
|
stdout, stderr = process.communicate()
|
||||||
process.communicate()
|
|
||||||
stdout = ""
|
stdout = stdout.decode()
|
||||||
stderr = ""
|
stderr = stderr.decode()
|
||||||
|
|
||||||
|
# process.communicate()
|
||||||
|
# stdout = ""
|
||||||
|
# stderr = ""
|
||||||
# print(stderr.decode())
|
# print(stderr.decode())
|
||||||
|
|
||||||
return (process.returncode, stdout, stderr)
|
return process.returncode, stdout, stderr
|
||||||
|
|
||||||
#
|
#
|
||||||
def ExecuteRsync(self, _args):
|
def execute_rsync(self, _args):
|
||||||
|
|
||||||
# Demand stuff
|
# Demand stuff
|
||||||
self.DemandDestinationDirectoryConfig()
|
self.demand_source_directory_config()
|
||||||
if self.IsUsingSSH():
|
self.demand_destination_directory_config()
|
||||||
self.DemandSSHConfig()
|
if self.is_using_ssh():
|
||||||
|
self.demand_ssh_config()
|
||||||
|
|
||||||
#
|
#
|
||||||
args = self.StartRsyncArgs()
|
args = self.start_rsync_args()
|
||||||
args.extend(_args)
|
args.extend(_args)
|
||||||
# print(str(args))
|
# print(str(args))
|
||||||
|
|
||||||
#
|
#
|
||||||
env = self.StartRsyncEnvironmentalVariables()
|
env = self.start_rsync_environment_variables()
|
||||||
|
|
||||||
|
#
|
||||||
|
self.log("Executing rsync with the following arguments:", args)
|
||||||
|
self.log("; And the following environment:", env)
|
||||||
|
|
||||||
#
|
#
|
||||||
# print("Debug -> Want to execute Rsync")
|
# print("Debug -> Want to execute Rsync")
|
||||||
@ -475,16 +798,24 @@ class MikesBackup:
|
|||||||
stdout = ""
|
stdout = ""
|
||||||
stderr = ""
|
stderr = ""
|
||||||
|
|
||||||
return (process.returncode, stdout, stderr)
|
# Check return code
|
||||||
|
# 0 = Success
|
||||||
|
# 24 = Source files vanished
|
||||||
|
return_code = process.returncode
|
||||||
|
if return_code != 0 and return_code != 24:
|
||||||
|
raise Exception("Rsync seems to have failed somehow! Got return code: " + str(return_code))
|
||||||
|
|
||||||
|
return return_code, stdout, stderr
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
|
||||||
|
b = MikesBackup()
|
||||||
|
b.do_backup()
|
||||||
|
|
||||||
|
|
||||||
#
|
#
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
#
|
#
|
||||||
b = MikesBackup()
|
main()
|
||||||
b.DoBackup()
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Reference in New Issue
Block a user