Python, want logging with log rotation and compression

Question

Can anyone suggest a way in python to do logging with:

• log rotation every day
• compression of logs when they\'re rotated
• optional - delete old

Answer 1

I think that the best option will be to use current implementation of TimedRotatingFileHandler and after renaming log file to the rotated version just compress it:

import zipfile
import os
from logging.handlers import TimedRotatingFileHandler


class TimedCompressedRotatingFileHandler(TimedRotatingFileHandler):
    """
    Extended version of TimedRotatingFileHandler that compress logs on rollover.
    """
    def find_last_rotated_file(self):
        dir_name, base_name = os.path.split(self.baseFilename)
        file_names = os.listdir(dir_name)
        result = []
        prefix = '{}.20'.format(base_name)  # we want to find a rotated file with eg filename.2017-12-12... name
        for file_name in file_names:
            if file_name.startswith(prefix) and not file_name.endswith('.zip'):
                result.append(file_name)
        result.sort()
        return result[0]

    def doRollover(self):
        super(TimedCompressedRotatingFileHandler, self).doRollover()

        dfn = self.find_last_rotated_file()
        dfn_zipped = '{}.zip'.format(dfn)
        if os.path.exists(dfn_zipped):
            os.remove(dfn_zipped)
        with zipfile.ZipFile(dfn_zipped, 'w') as f:
            f.write(dfn, dfn_zipped, zipfile.ZIP_DEFLATED)
        os.remove(dfn)

Answer 2

I have added below one solution where i am basically zipping old backup logs to zip with timestamp on it. using one extra variable called ZipbackupCount. # no of old files to be zipped

e.g. we have logs like this. (backupcount = 5 and ZipbackupCount = 2)

a.log.1

a.log.2

a.log.3

a.log.4

a.log.11-09-2020-11-11-11.zip

once count of backup logs hits to 5 it triggers to zip a.log.5 and a.log.4 to above zip and continues.

import os
import datetime
import gzip
import logging.handlers
import zipfile

from config.config import PROJECT_PATH, LOG_DIR, LOG_FILE_NAME, LOG_FILESIZE


class NewRotatingFileHandler(logging.handlers.RotatingFileHandler):
    def __init__(self, filename, **kws):
        backupCount = kws.get('backupCount', 0)
        self.backup_count = backupCount
        self.ZipbackupCount = kws.pop('ZipbackupCount', 0)
        self.file_name = filename
        self.log_dir = os.path.split(self.file_name)[0]
        self.log_file_name = os.path.split(self.file_name)[-1]

        logging.handlers.RotatingFileHandler.__init__(self, filename, **kws)

    def doArchive(self, old_log):
        with open(old_log) as log:
            with gzip.open(old_log + '.gz', 'wb') as comp_log:
                comp_log.writelines(log)

        os.remove(old_log)

    def doRollover(self):
        super(NewRotatingFileHandler, self).doRollover()

        zip_file_name = self.log_file_name + "." + datetime.datetime.now().strftime("%d-%m-%Y-%H-%M-%S") + ".zip"

        if os.path.exists(self.rotation_filename("%s.%d" % (self.baseFilename, self.backupCount))) and self.ZipbackupCount > 0 and self.file_name:

            with zipfile.ZipFile(os.path.join(self.log_dir, zip_file_name), "w", zipfile.ZIP_DEFLATED, allowZip64=True) as zf:

                for i in range(self.backupCount, self.backupCount - self.ZipbackupCount, -1):

                    sfn = self.rotation_filename("%s.%d" % (self.baseFilename, i))

                    if os.path.exists(sfn):
                        zf.write(sfn, "%s.%d" % (self.log_file_name, i))
                        os.remove(sfn)
                    else:
                        continue

                zf.close()


# handler = NewRotatingFileHandler(filename=os.path.join(PROJECT_PATH, LOG_DIR, LOG_FILE_NAME),
#                                  maxBytes=LOG_FILESIZE, backupCount=5, ZipbackupCount=2)
#
# handler.doRollover()

Answer 3

The other way to compress logfile during rotate (new in python 3.3) is using BaseRotatingHandler (and all inherited) class attribute rotator for example:

import gzip
import os
import logging
import logging.handlers

class GZipRotator:
    def __call__(self, source, dest):
        os.rename(source, dest)
        f_in = open(dest, 'rb')
        f_out = gzip.open("%s.gz" % dest, 'wb')
        f_out.writelines(f_in)
        f_out.close()
        f_in.close()
        os.remove(dest)

logformatter = logging.Formatter('%(asctime)s;%(levelname)s;%(message)s')
log = logging.handlers.TimedRotatingFileHandler('debug.log', 'midnight', 1, backupCount=5)
log.setLevel(logging.DEBUG)
log.setFormatter(logformatter)
log.rotator = GZipRotator()

logger = logging.getLogger('main')
logger.addHandler(log)    
logger.setLevel(logging.DEBUG)

....

More you can see here.