import os.path import docker import restic import yaml import configparser import argparse bindingIncludePattern = None bindingExcludePattern = None remoteHost = None remoteUser = None repository = None passwordFile = None def arguments_parser(): ''' récupère les arguments de la commande :return: object with arguments as members ''' parser = argparse.ArgumentParser( prog='BackandUp', description='Parse your docker-compose.yml file to automatically shut the containers, backup their volumes ' 'and restart them. Dependencies between containers are respected.', epilog='BackandUp 2023') parser.add_argument('-c', '--config', action='store', default='./configuration.ini', help='configuration file for BackandUp. By default it looks for a configuration.ini ' 'in its folder. It must contains the configuration for the backup service ' 'and can also contains the BackandUp own configuration.') parser.add_argument('--compose', action='store', default='./docker-compose.yml', help='compose file to parse. By default it looks for a docker-compose.yml in its folder.') parser.add_argument('--sshfolder', action='store', default='~/.ssh', help='ssh folder for sftp connection. By default it uses ~/.ssh.') parser.add_argument('-x', '--bindexclude', action='store', default=None, help='Patterns of volume or bind to exclude from the backup.') parser.add_argument('-i', '--bindinclude', action='store', default=None, help='Patterns of volume or bind to include from the backup. Using this argument, ' 'only matching volumes and binds will be backed up.') return parser.parse_args() def read_config_file(filepath: str = "configuration.ini"): parser = configparser.ConfigParser() parser.read(filepath) global arguments if "backandup" in parser: if "composefile" in parser["backandup"]: arguments.compose = parser["backandup"]["composefile"] if "composefile" in parser["backandup"]: arguments.sshfolder = parser["backandup"]["sshfolder"] if "composefile" in parser["backandup"]: arguments.bindinclude = parser["backandup"]["bindincludepattern"] if "composefile" in parser["backandup"]: arguments.bindexclude = parser["backandup"]["bindexcludepattern"] if "restic" in parser: arguments.remotehost = parser["restic"]["remotehost"] arguments.repository = parser["restic"]["repository"] arguments.passwordfile = parser["restic"]["passwordfile"] if "dryrun" in parser["restic"]: arguments.dryrun = (parser["restic"]["dryrun"].lower() == "true" or parser["restic"]["dryrun"] > 0) else: arguments.dryrun = False def restic_backup(dirpath: str, tagslist: list = None): ''' Lance la sauvegarde avec un container restic one-shot :param dirpath: str :param tags: list :return: ''' dockerhost = docker.from_env() ctpath = dirpath # si il s'agit d'un volume on ne peut pas deviner son point de montage alors on le met arbitrairement à la racine if not os.path.exists(dirpath): ctpath = f"/{ctpath}" tagscommand = "" if tagslist is not None and len(tagslist): for tag in tagslist: tagscommand = f"{tagscommand}--tag {tag} " tagscommand = tagscommand[:-1] dryrun = "" if arguments.dryrun: dryrun = "--dry-run" volumes = {dirpath: {'bind': dirpath, 'mode': 'ro'}, arguments.sshfolder: {'bind': '/root/.ssh', 'mode': 'ro'}, arguments.passwordfile: {'bind': "/repopassword", 'mode': 'ro'}} command = f"backup {dryrun} {tagscommand} {dirpath}" hostname = "restic" environment = ["RESTIC_PASSWORD_FILE=/repopassword", f"RESTIC_REPOSITORY={arguments.repository}"] dockerhost.containers.run("restic/restic", command=command, environment=environment, volumes=volumes, remove=True, hostname=hostname) def yaml_to_services_list(filepath: str) -> dict|None: ''' Retourne la liste des services définis dans un fichier docker-compose.yaml sous forme de liste :param filepath: str :return: list|None ''' print(f"sorting services from {filepath}") serviceslist = dict() if not os.path.isfile(filepath): raise OSError(2, "No such file", filepath) ycontent = None with open(filepath, 'r') as yfile: try: ycontent = yaml.safe_load(yfile) except yaml.YAMLError as yerror: print(f"{filepath} doesn't seems to be a yaml file.") raise yerror if ycontent is None: return None if "services" not in ycontent: raise Exception("The file doesn't seems to be a correct docker-compose yaml") # on liste les services et ceux dont ils dépendent for servicename in ycontent["services"]: print(f"appending {servicename}") mustbefore = [] if "depends_on" in ycontent["services"][servicename]: for dependencie in ycontent["services"][servicename]["depends_on"]: mustbefore.append(dependencie) serviceslist[servicename] = {"must_before": mustbefore, "must_after": [], "backupdone": False} # on reparcourt une seconde fois pour définir de quels services ilssont les dépendances for service in serviceslist: if len(serviceslist[service]["must_before"]): for dependencie in serviceslist[service]["must_before"]: if service not in serviceslist[dependencie]["must_after"]: serviceslist[dependencie]["must_after"].append(service) return serviceslist def order_services_by_dependencies(servicesdict: dict) -> list: ''' Génère une liste avec les noms de services pour ordonner leurs coupures par dépendances. Les containers étant requis seront placés avant ceux les requérant. :param servicesdict: dict :return: list ''' serviceslist = [] for service in servicesdict: if len(servicesdict[service]["must_after"]) and len(serviceslist): depindex = len(serviceslist) -1 for dependencieof in servicesdict[service]["must_after"]: if dependencieof in serviceslist and serviceslist.index(dependencieof) < depindex: depindex = serviceslist.index(dependencieof) serviceslist.insert(depindex, service) else: serviceslist.append(service) return serviceslist def backup_services(servicename: str, servicesdict: dict, composefilepath: str) -> bool: ''' Pause les containers, lance la backup des dépendances si il y en a, lance sa propre backup, puis redémarre les containers. :param servicename: str :param servicesdict: dict :param composefilepath: str :return: bool ''' print(f"Starting process for {servicename}:") if servicesdict[servicename]["backupdone"]: print(f" backup is already done, ignoring") return True dockerclient = docker.from_env() containersid = os.popen(f"docker-compose --file {composefilepath} ps --quiet {servicename}").read().splitlines() containerrunning = [] for containerid in containersid: container = dockerclient.containers.get(containerid) containerrunning.append(container.status == "running") print(f" stopping {container.name}") container.stop() if len(servicesdict[servicename]["must_before"]): for dependencie in servicesdict[servicename]["must_before"]: backup_services(dependencie, servicesdict, composefilepath) # TODO: faire la sauvegarde for containerid in containersid: container = dockerclient.containers.get(containerid) if "Binds" in container.attrs["HostConfig"] and len(container.attrs["HostConfig"]["Binds"]): for bind in container.attrs["HostConfig"]["Binds"]: bindpath = bind.split(":")[0] print(f" backup of {bindpath}") restic_backup(dirpath=bindpath, tagslist=[servicename]) for containerindex in range(0, len(containersid)): if containerrunning[containerindex]: container = dockerclient.containers.get(containersid[containerindex]) print(f" restarting {container.name}") container.start() servicesdict[servicename]["backupdone"] = True return True # Press the green button in the gutter to run the script. if __name__ == '__main__': dockerhost = docker.from_env() arguments = arguments_parser() servicesDict = yaml_to_services_list(arguments.compose) servicesList = order_services_by_dependencies(servicesDict) print("Backup process is starting") for servicetobackup in reversed(servicesList): backup_services(servicetobackup, servicesDict, arguments.compose)