#!/bin/python3 import os, subprocess, yaml, time, socket, datetime, getopt, sys, re, atexit, logging as log from pathlib import Path help_text = '' def main(args): # arg-settings arg_short_options = "hc:fv" arg_long_options = ["help", "config=", "cmd=", "force", "verbose"] global help_text help_text = f""" Usage: {args[0]} [options]\n Options: -h --help Display this text -c --config Specify the config-file default='backup.conf' --cmd Run command for repository -f --force Force backup (ignore all conditions)""" args = readArgs( args = args[1:], short_options = arg_short_options, long_options = arg_long_options ) args = parseCliArgs( args = args, data = { 'config': "backup.conf", 'force': False, 'verbose': False }, argMap = { 'config': ['config', 'c'], 'cmd': ['cmd'], 'force': ['force', 'f'], "verbose": ['verbose', 'v'] } ) setupLogging(args['verbose']) config = readConfig(args['config']) setDefaultValues(config, args['config']) # Execute cmd if requested if "cmd" in args: return executeBorgCommand(config, args['cmd']) # Get config data include = config["backup"]["include"] exclude = config["backup"]["exclude"] if not args['force']: # Pre-backup-checks result = checkRunConditions(config) if not (isinstance(result, bool) and result): log.error(result) return 0 # Previous-backup-metadata metadata = readBackupMetadata(config) if metadata: handleBackupMetadata(config, metadata) # Folder-scan if config["backup"]["scan"]["enabled"]: includeScanned, excludeScanned = scanForFolders(config) # Touch files if requested if config["backup"]["scan"]["touch"]: touchFiles(includeScanned+excludeScanned) # PRE - Execute files if requested if config["backup"]["scan"]["execute"]: executeFilesPre(includeScanned+excludeScanned) # Register post at exit atexit.register(executeFilesPost, (includeScanned+excludeScanned)) # Add folders containing these files to include/exclude include += [os.path.dirname(file) for file in includeScanned] exclude += [os.path.dirname(file) for file in excludeScanned] command = generateCommand(config, include, exclude) runBackup(config, command) return 0 def setDefaultValues(config, configFile): if config['backup']['metadata']['file'] == '': config['backup']['metadata']['file'] = f"{configFile}.data" if config['backup']['scan']['cache']['file'] == '': config['backup']['scan']['cache']['file'] = f"{configFile}.cache" def readArgs(args, short_options: str, long_options: list): try: arguments, values = getopt.getopt(sys.argv[1:], short_options, long_options) return arguments except getopt.error as err: print (str(err)) sys.exit(2) def parseCliArgs(args, data={}, argMap={}): # If argument-Map is empty, construct it if argMap == {}: for data_key, data_value in data: argMap[data_key] = [data_key] # Search in given arguments for argMap and write values for arg, val in args: # Special arguments if arg in ("-h", "--help"): global help_text print (help_text) sys.exit(0) found = False for map_key in argMap: map_altKeys = argMap[map_key] for altKey in map_altKeys: keyName = f"-{altKey}" # If name is longer, add second - for long-arg if len(altKey) > 1: keyName = f"-{keyName}" # If argument matches if arg == keyName: # If data expects bool write True, otherwise just write given value # and stop for this arg if map_key in data and isinstance(data[map_key], bool): data[map_key] = True else: data[map_key] = val found = True break if found: break return data def setupLogging(verbose: bool): level = log.INFO format = '%(asctime)s %(levelname)s: %(message)s' if verbose: level = log.DEBUG log.basicConfig(stream=sys.stdout, format=format, level=level) def readConfig(file: str): log.debug(f"Reading config '{file}'..") if not os.path.isfile(file): raise OSError(2, file) return readYamlFile(file) def readYamlFile(file: str): with open(file, "r") as f: return yaml.load(f, Loader=yaml.FullLoader) def checkRunConditions(config): log.debug("checkRunConditions") result = checkBatteryCharge(config) if not (isinstance(result, bool) and result): return result result = checkNetwork(config) if not (isinstance(result, bool) and result): return result return True def checkBatteryCharge(config): log.debug("> Battery-charge") if not config["backup"]["run-conditions"]["battery"]["min-percent"] > 0: log.debug("Check disabled, skipping") return True # Stats-file-stream exists? if not os.path.exists("/sys/class/power_supply/battery/capacity"): log.warning("No battery found, skipping") return True # Read with open("/sys/class/power_supply/battery/capacity", "r") as file: battery_charge = int(file.readlines()[0]) if battery_charge < config["backup"]["run-conditions"]["battery_min-percent"]: log.debug(f"Too low ({battery_charge} < {config['backup']['run-conditions']['battery']['min-percent']})") errorMsg = f"Battery-charge too low ({battery_charge} < {config['backup']['run-conditions']['battery']['min-percent']})" log.debug("or-AC-connected") if config["backup"]["run-conditions"]["battery"]["or_ac-connected"]: # Check battery-status battery_status = "Unknown" with open("/sys/class/power_supply/battery/status", "r") as file: battery_status = file.readlines()[0].strip() if not battery_status == "Charging": log.debug("Not charging ("+ battery_status +")") errorMsg += " and not charging" return errorMsg else: log.debug(f"Ok ({battery_charge} < {config['backup']['run-conditions']['battery']['min-percent']})") return True def checkNetwork(config): log.debug("Check Network") hostname = getHostnameFromRepository(config["borg"]["repository"]) if hostname == None: log.debug("Repository does not have a hostname, skipping") return True result = checkNetworkHops(config, hostname) if not (isinstance(result, bool) and result): return result return True def checkNetworkHops(config, hostname: str): log.debug("> Network-hops") if not config["backup"]["run-conditions"]["network"]["max_hops"] > 0: log.debug("Check disabled (0)") return True ip = resolveHostname(hostname) lastHopIp = traceroute( maxhops = config["backup"]["run-conditions"]["network"]["max_hops"], ip = ip )[-2].strip() if(len(lastHopIp) >= 7): lastHopIp = lastHopIp.split(" ")[2] log.debug(f"LastHop-IP: {lastHopIp}") if ip != lastHopIp: return "Could not reach server in <= "+ str(config['backup']['run-conditions']['network']['max_hops']) +" hops" return True def getHostnameFromRepository(repository: str): # Check if has hostname result = re.search(r'^(\w+@)?(([A-Z0-9-]+\.)+[A-Z0-9-]+):.*$', repository, flags= re.I) if result: return result[2] else: # No hostname return def resolveHostname(hostname: str): log.debug(f"Resolving hostname {hostname}..") ip = socket.gethostbyname(hostname) log.debug(f"IP: {ip}") return ip def traceroute(maxhops: int, ip: str): command = ["traceroute", "-n", f"-m {maxhops}", "-q 2", ip] log.debug(f"Running traceroute to ip={ip} with maxhops={maxhops}..") result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) out = result.stdout.decode('utf-8').split('\n') log.debug(out) return out def readBackupMetadata(config): file = config['backup']['metadata']['file'] if os.path.isfile(file): log.debug(f"Reading metadata-file '{file}'..") return readYamlFile(file) return def writeBackupMetadata(config, success: bool): data = { "time": int(time.time()), "success": success } log.debug(f"Writing metadata: {data}") with open(config['backup']['metadata']['file'], 'w') as file: yaml.dump(data, file) def handleBackupMetadata(config, metadata): log.debug(f"Handling metadata..") age = int(time.time()-metadata["time"]) if age <= config["backup"]["backup-age"]: if metadata["success"]: log.info("Backup-age is "+ str(age) +"s, next backup will be created in "+ str(age+config["backup"]["backup-age"]) +"s") quit() else: log.warning("Backup-age is "+ str(age) +"s, but was not successful!") def scanForFolders(config): log.debug(f"Scan for marked folders") includeScannedFiles = [] excludeScannedFiles = [] # Read cache if exists if(os.path.isfile(config["backup"]["scan"]['cache']['file'])): log.debug(f"Cache-file found") cacheData = readYamlFile(config["backup"]["scan"]['cache']['file']) age = int(time.time()-cacheData["time"]) log.debug(f"Cache-age: {str(age)}") if age <= config["backup"]["scan"]["cache"]["valid-time"]: log.info("Using Cache ("+ str(age) +"s old)") scan = False includeScannedFiles = cacheData["include"] excludeScannedFiles = cacheData["exclude"] log.debug(f"include: {includeScannedFiles}") log.debug(f"exclude: {excludeScannedFiles}") return includeScannedFiles, excludeScannedFiles # Scan folders for files log.info(f"Scanning for folders marked with {config['backup']['scan']['backup']} or {config['backup']['scan']['nobackup']}..") for location in config["backup"]["scan"]["locations"]: log.debug(f"Scanning {location}..") includeScannedFile = find(location, config["backup"]["scan"]["backup"]) excludeScannedFile = find(location, config["backup"]["scan"]["nobackup"]) if len(includeScannedFile) > 0: log.debug(f"include: {includeScannedFile}") if len(excludeScannedFile) > 0: log.debug(f"exclude: {excludeScannedFile}") includeScannedFiles += includeScannedFile excludeScannedFiles += excludeScannedFile # Write cache log.debug("Writing cache..") with open(config["backup"]["scan"]['cache']['file'], 'w') as file: yaml.dump( { "time": int(time.time()), "include": includeScannedFiles, "exclude": excludeScannedFiles } , file) log.info(f"Scan finished. {len(includeScannedFiles) + len(excludeScannedFiles)} files found") log.debug(f"include: {includeScannedFiles}") log.debug(f"exclude: {excludeScannedFiles}") return includeScannedFiles, excludeScannedFiles def touchFiles(files: list): for file in files: if os.path.isfile(file): os.utime(file, None) def executeFilesPre(files: list): log.info("Running script-pre..") # Store files which were successfully executed executedFiles = [] try: for file in files: # Check for filesize and execution-permission if os.stat(file).st_size != 0 and os.access(file, os.X_OK): log.info(f"{file}") subprocess.run([file, "pre"], check=True) executedFiles += file except subprocess.CalledProcessError as err: log.critical("Script returned error, cannot continue!") executeFilesPost(executedFiles) raise err def executeFilesPost(files: list): log.info("Running script-post..") for file in files: # Check for filesize and execution-permission if os.stat(file).st_size != 0 and os.access(file, os.X_OK): log.info(f"{file}") try: subprocess.run([file, "post"], check=True) except subprocess.CalledProcessError as err: log.error(err) pass def generateEnviromentVars(config): args = config["borg"]["args"] envVars = [env_var +"='"+ config["borg"]["env-vars"][env_var] +"'" for env_var in config["borg"]["env-vars"]] return ' '.join(envVars) def generateCommand(config, include, exclude): # Generate arguments folderArgs = [f"'{folder}'" for folder in include] folderArgs += [f"--exclude '{folder}'" for folder in exclude] folderArgs = ' '.join(folderArgs) envVars = generateEnviromentVars(config) compression = config["borg"]["compression"] repository = config['borg']['repository'] backupName = socket.getfqdn() +"__"+ datetime.datetime.now().strftime('%F_%T') return f"{envVars} borg create {config['borg']['args']} --exclude-caches --compression {compression} {repository}::{backupName} {folderArgs}" def executeBorgCommand(config, command): envVars = generateEnviromentVars(config) repository = config['borg']['repository'] borgSubcommand = command commandArgs = '' if ' ' in command: commandData = command.split(' ') borgSubcommand = commandData[0] commandArgs = ' '.join(commandData[1:]) command = f"{envVars} borg {borgSubcommand} {repository} {commandArgs}" log.debug(f'Executing {command}') return os.system(command) def runBackup(config, command): writeBackupMetadata(config, False) log.info("Running backup") for i in range(1, config["backup"]["tries"]["amount"]): log.debug(f"Try {i}") log.debug(command); code = os.system(command) log.debug(f"Return-code {code}") if not code == 0: log.error("Backup failed") if(i+1 != config["backup"]["tries"]["amount"]): print("- Retry in "+ str(config["backup"]["tries"]["sleep"]) +"s") time.sleep(config["backup"]["tries"]["sleep"]) else: log.info("Backup successful") writeBackupMetadata(config, True) return log.critical("Backup completly failed!") def find(path: str, searchName: str): result = subprocess.run(["find", path, "-type", "f", "-name", searchName], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) return result.stdout.decode('utf-8').split('\n')[:-1] def get_platform(): platforms = { 'linux1' : 'Linux', 'linux2' : 'Linux', 'darwin' : 'OS X', 'win32' : 'Windows' } if sys.platform not in platforms: return sys.platform return platforms[sys.platform] if __name__ == "__main__": sys.exit(main(sys.argv))