Default configuration

CNFG

import pycnfg
import sklearn
import mlshell

__all__ = ['CNFG']


PATHS = {
    'init': pycnfg.find_path,
    'producer': pycnfg.Producer,
    'priority': 1,
    'steps': [],
    'default': {},
}


LOGGERS = {
    'init': 'default',
    'producer': mlshell.LoggerProducer,
    'priority': 2,
    'steps': [
        ('make',),
    ],
    'default': {},
}


PIPELINES = {
    'init': mlshell.Pipeline,
    'producer': mlshell.PipelineProducer,
    'priority': 3,
    'steps': [
        ('make', ),
    ],
}


METRICS = {
    'init': mlshell.Metric,
    'producer': mlshell.MetricProducer,
    'priority': 4,
    'steps': [
        ('make', ),
    ],
}


DATASETS = {
    'init': mlshell.Dataset,
    'producer': mlshell.DatasetProducer,
    'priority': 5,
    'steps': [
        ('load', ),
        ('info', ),
        ('preprocess', ),
        ('split',),
    ],
}

WORKFLOWS = {
    'init': {},
    'producer': mlshell.Workflow,
    'priority': 6,
    'steps': [
        ('optimize', ),
        ('validate', ),
        ('predict', ),
        ('dump', ),
    ],
}


CNFG = {
    'path': PATHS,
    'logger': LOGGERS,
    'pipeline': PIPELINES,
    'dataset': DATASETS,
    'metric': METRICS,
    'workflow': WORKFLOWS,
}
"""Default sections for ML task."""

logger

class LevelFilter(object):
    """Custom filter."""
    def __init__(self, level=50):
        self._level = level

    def filter(self, record):
        if record.levelno <= self._level:
            return True
        else:
            return False


class CustomFormatter(logging.Formatter):
    def format(self, record):
        s = super().format(record)
        s = s.replace('    |__ ', '     \u25B6 ')
        s = s.replace('|__ ', '\u25CF ')
        return s


LOGGER_CONFIG = {
    "version": 1,
    'filters': {
        'test_level_filter': {
            '()': LevelFilter,
            'level': 5,
        }
    },
    "handlers": {
        "test_handler": {
            "class": "logging.FileHandler",
            "formatter": "custom",
            "filename": 'test.log',
            "level": 5,
            "filters": ['test_level_filter']
        },
        "debug_handler": {
            "class": "logging.handlers.RotatingFileHandler",
            "maxBytes": 25000000,  # 25mb 25*10^6 byte.
            "backupCount": 3,      # Amount of backup files.
            "formatter": "custom",
            "filename": 'debug.log',  # Init only, will be replaced.
            "level": "DEBUG",
        },
        "info_handler": {
            "class": "logging.handlers.RotatingFileHandler",
            "maxBytes": 25000000,
            "backupCount": 5,
            "formatter": "custom",
            "filename": 'info.log',
            "level": "INFO",  # 20
        },
        "minimal_handler": {
            "class": "logging.handlers.RotatingFileHandler",
            "maxBytes": 25000000,
            "backupCount": 5,
            "formatter": "custom",
            "filename": 'info.log',
            "level": 25,
        },
        "warning_handler": {
            "class": "logging.FileHandler",
            "formatter": "default",
            "filename": 'warning.log',
            "level": "WARNING",  # 30
        },
        "error_handler": {
            "class": "logging.FileHandler",
            "formatter": "custom",
            "filename": 'error.log',
            "level": "ERROR",
        },
        "critical_handler": {
            "class": "logging.FileHandler",
            "formatter": "custom",
            "filename": 'critical.log',
            "level": "CRITICAL",
        },
        "console_handler": {
            "class": "logging.StreamHandler",
            "formatter": "custom",
            "level": "INFO",
            "stream": None,  # Default in std.err (marked).
        },
        "http_handler": {
            "class": "logging.handlers.HTTPHandler",
            "formatter": "custom",
            "level": "ERROR",
            "host": 'www.example.com',
            "url": 'https://wwww.example.com/address',
            "method": "POST",
        },
    },
    "loggers": {
        "default": {
            "handlers": ["test_handler", "debug_handler", "info_handler",
                         "minimal_handler", "warning_handler", "error_handler",
                         "critical_handler", "console_handler", "http_handler"
                         ],
            "level": 1,  # => use handlers levels.
        },
    },
    "formatters": {
        "default": {
            "format": "%(levelname)s: %(message)s%(delimeter)s"
        },
        "custom": {
            "()": CustomFormatter,
            "format": "%(message)s%(delimeter)s",
        },
    }
}
"""dict : Logger configuration for logging.config.dictConfig method.

Levels of logging:

* critical
    reset on logger creation.
* error
    reset on logger creation.
* warning
    reset on logger creation.
* minimal
    cumulative.
    only score for best run in gs.
* info
    cumulative.
    workflow information.
* debug
    reset on logger creation.
    detailed workflow information.
* test
    only for test purposes (will be created only in pytest env).
        
"""