#!/usr/bin/env python3
# Author(s): Toni Sissala
# Copyright 2019 Finnish Social Science Data Archive FSD / University of Tampere
# Licensed under the EUPL. See LICENSE.txt for full license.
"""Command line setup for Kuha applications
Parse command line for common configuration options
and store loaded settings.
Load modules for querying Document Store::
import os
from kuha_common import cli_setup
cli_setup.load(os.getcwd())
settings = cli_setup.setup(cli_setup.MOD_DS_CLIENT, cli_setup.MOD_DS_QUERY)
"""
import os
import logging
import configargparse
from kuha_common.document_store.query import Query
from kuha_common.document_store.client import JSONStreamClient
from kuha_common.document_store.constants import (
DS_CLIENT_MAX_CLIENTS,
DS_CLIENT_REQUEST_TIMEOUT,
DS_CLIENT_CONNECT_TIMEOUT
)
ABS_DIR_PATH = os.path.dirname(os.path.realpath(__file__))
ABS_CWD_PATH = os.getcwd()
CFG_FILENAME = 'kuha.ini'
DS_API_VERSION = 'v0'
LOG_FORMAT = '%(asctime)s %(levelname)s(%(name)s): %(message)s'
LOG_LEVELS = [
'CRITICAL',
'ERROR',
'WARNING',
'INFO',
'DEBUG'
]
#: Constant for configuring :mod:`kuha_common.document_store.client`
MOD_DS_CLIENT = 'document_store.client'
#: Constant for configuring :mod:`kuha_common.document_store.query`
MOD_DS_QUERY = 'document_store.query'
#: Constant for configuring :mod:`logging`
MOD_LOGGING = 'logging'
[docs]def add_kuha_loglevel(parser=None):
"""Add loglevel to parser
:param parser: command line parser.
:type parser: ArgumentParser instance
"""
parser = parser or settings.parser
parser.add('--loglevel',
help='Lowest logging level of log messages to output',
default='INFO',
choices=LOG_LEVELS,
env_var='KUHA_LOGLEVEL',
type=str)
[docs]def add_document_store_url(parser=None, **kw):
r"""Add document store url to parser
:param parser: command line parser.
:type parser: ArgumentParser instance
:param \*\*kw: keyword arguments get passes to parser.add
"""
parser = parser or settings.parser
parser.add('--document-store-url',
help=('Full URL of Kuha document store database. Clients may use this '
'instead of configuration options --document-store-host, '
'--document-store-port and --document-store-api-version.'),
env_var='KUHA_DS_URL', type=str, **kw)
[docs]def add_document_store_host(parser=None):
"""Add document store host to parser
:param parser: command line parser.
:type parser: ArgumentParser instance
"""
parser = parser or settings.parser
parser.add('--document-store-host',
help=('Host of Kuha document store database.'),
default='http://localhost',
env_var='KUHA_DS_HOST',
type=str)
[docs]def add_document_store_port(parser=None):
"""Add document store port to parser
:param parser: command line parser.
:type parser: ArgumentParser instance
"""
parser = parser or settings.parser
parser.add('--document-store-port',
help=('Port of Kuha document store database.'),
default=6001,
env_var='KUHA_DS_PORT',
type=int)
[docs]def add_document_store_api_version(parser=None):
"""Add document store api-version to parser
:param parser: command line parser.
:type parser: ArgumentParser instance
"""
parser = parser or settings.parser
parser.add('--document-store-api-version',
help=('Document Store api version.'),
default=DS_API_VERSION,
env_var='KUHA_DS_API_VERSION',
type=str)
[docs]def add_document_store_client_request_timeout(parser=None):
"""Add document store client request timeout to parser
:param parser: command line parser.
:type parser: ArgumentParser instance
"""
parser = parser or settings.parser
parser.add('--document-store-client-request-timeout',
help='Configure the request timeout (in seconds) for Document Store client.',
default=DS_CLIENT_REQUEST_TIMEOUT,
env_var='KUHA_DOCUMENT_STORE_CLIENT_REQUEST_TIMEOUT',
type=int)
[docs]def add_document_store_client_connect_timeout(parser=None):
"""Add document store client connect timeout to parser
:param parser: command line parser
:type parser: ArgumentParser instance
"""
parser = parser or settings.parser
parser.add('--document-store-client-connect-timeout',
help='Configure the connect timeout (in seconds) for Document Store client.',
default=DS_CLIENT_CONNECT_TIMEOUT,
env_var='KUHA_DOCUMENT_STORE_CLIENT_CONNECT_TIMEOUT',
type=int)
[docs]def add_document_store_client_max_clients(parser=None):
"""Add document store client max clients timeout to parser
:param parser: command line parser.
:type parser: ArgumentParser instance
"""
parser = parser or settings.parser
parser.add('--document-store-client-max-clients',
help='Configure maximum for simultaneous client connections for Document Store client.',
default=DS_CLIENT_MAX_CLIENTS,
env_var='KUHA_DOCUMENT_STORE_CLIENT_MAX_CLIENTS',
type=int)
[docs]def add_print_configuration(parser=None):
"""Add print configuration helper for testing configuration options.
:param parser: command line parser.
:type parser: ArgumentParser instance
"""
parser = parser or settings.parser
parser.add('--print-configuration',
help='Print loaded configuration and exit.',
default=False,
action='store_true')
[docs]def add_server_process_count_configuration(parser=None):
"""Add tornado server process count to configuration options parser.
:param parser: command line parser.
:type parser: ArgumentParser instance
"""
parser = parser or settings.parser
parser.add('--server-process-count',
help='Set number of processes that tornado server will fork. '
'0 forks one process for each processor core.',
default=0, type=int, env_var='KUHA_SERVER_PROCESS_COUNT')
[docs]class KuhaConfigFileParser(configargparse.DefaultConfigFileParser):
"""Inherit to override :meth:`configargparse.DefaultConfigFileParser.get_syntax_description`
"""
[docs] def get_syntax_description(self):
"""Override syntax description of :class:`configargparse.DefaultConfigFileParser`
:returns: Syntax description for configuration file.
:rtype: str
"""
msg = ("Config file syntax allows: key=value, flag=true, stuff=[a,b,c], "
"and ignores sections ([section]), so all configuration options should "
"be considered global in application context. Note that not all "
"configuration options, although supported, make sense in "
"configuration file, for example help-option.")
return msg
[docs]class Settings:
"""Class for command line settings."""
def __init__(self):
self.settings = None
self.parser = None
self.abs_dir_path = None
[docs] def is_parser_loaded(self):
"""Check is parser loaded.
:rtype: bool
"""
return self.parser is not None
[docs] def is_settings_loaded(self):
"""Check is settings loaded.
:rtype: bool
"""
return self.settings is not None
[docs] def set_abs_dir_path(self, path):
"""Set absolute directory path of configurable kuha application.
:param path: absolute path to kuha application directory.
:type path: str
"""
self.abs_dir_path = path
[docs] def get_abs_dir_path(self):
"""Return absolute directory path of configurable kuha application.
:returns: absolute path to directory.
:rtype: str
"""
return self.abs_dir_path
[docs] def add_logging_configs(self):
"""Wrapper to add logging-module configuration."""
add_kuha_loglevel(self.parser)
add_kuha_logformat(self.parser)
[docs] def add_document_store_query_configs(self):
"""Wrapper to add document_store_query configuration."""
add_document_store_host(self.parser)
add_document_store_port(self.parser)
add_document_store_api_version(self.parser)
add_document_store_url(self.parser)
[docs] def add_document_store_client_configs(self):
"""Wrapper to add document_store_client configuration."""
add_document_store_client_request_timeout(self.parser)
add_document_store_client_connect_timeout(self.parser)
add_document_store_client_max_clients(self.parser)
[docs] def setup_logging(self):
"""Setup :mod:`logging` module."""
logging.basicConfig(level=getattr(logging, self.settings.loglevel),
format=self.settings.logformat)
[docs] def setup_document_store_query(self):
"""Setup :mod:`kuha_common.document_store.query` module."""
base_url = self.settings.document_store_url or '%s:%s/%s'\
% (self.settings.document_store_host,
self.settings.document_store_port,
self.settings.document_store_api_version)
Query.set_base_url(base_url)
[docs] def setup_document_store_client(self):
"""Setup :mod:`kuha_common.document_store.client` module."""
JSONStreamClient.set_max_clients(
self.settings.document_store_client_max_clients
)
JSONStreamClient.set_request_timeout(
self.settings.document_store_client_request_timeout
)
JSONStreamClient.set_connect_timeout(
self.settings.document_store_client_connect_timeout
)
[docs] def load_parser(self, config_file=None, **kw):
"""Load command line parser.
Additional keyword arguments are passed to
:class:`configargparse.ArgumentParser`.
:param config_file: Name of configuration file.
:type config_file: str
"""
if self.is_settings_loaded():
raise Exception("Configuration already loaded")
if self.is_parser_loaded():
raise Exception("Parser already loaded")
if config_file is None:
config_file = CFG_FILENAME
if self.abs_dir_path is None:
self.set_abs_dir_path(ABS_DIR_PATH)
self.parser = configargparse.ArgumentParser(
config_file_parser_class=KuhaConfigFileParser,
default_config_files=[
os.path.join(ABS_CWD_PATH, config_file),
os.path.join(self.abs_dir_path, config_file)
],
**kw
)
[docs] def set(self, parsed_opts):
"""Assign parser options to settings.
:param parsed_opts: parser options.
:type parsed_opts: :obj:`argparse.Namespace`
"""
self.settings = parsed_opts
[docs] def load_cli_args(self):
"""Load command line arguments."""
if not self.is_parser_loaded():
raise Exception("Parser not loaded")
if not self.is_settings_loaded():
self.set(self.parser.parse_args())
[docs] def get(self):
"""Return active settings.
:returns: active settings.
:rtype: :obj:`argparse.Namespace`
"""
return self.settings
[docs] def add(self, *args, **kwargs):
r"""Add item for parser.
Settings must not yet be loaded
but parser must be loaded.
:param \*args: arguments passed to ``configargparse.ArgumentParser``
:param \*\*kwargs: keyword arguments passed to ``configargparse.ArgumentParser``
"""
if self.is_settings_loaded():
raise Exception("Settings already loaded")
if not self.is_parser_loaded():
raise Exception("Parser not loaded")
self.parser.add(*args, **kwargs)
settings = Settings()
configurable_modules = {
MOD_DS_CLIENT: (settings.add_document_store_client_configs, settings.setup_document_store_client),
MOD_DS_QUERY: (settings.add_document_store_query_configs, settings.setup_document_store_query),
MOD_LOGGING: (settings.add_logging_configs, settings.setup_logging)
}
[docs]def setup(*modules):
r"""Setup command line parser.
Load modules, parse command line arguments, return
loaded settings in :obj:`argparse.Namespace`
:param \*modules: common Kuha modules to load and include in
parsing of command line arguments.
:type \*modules: str
:returns: Loaded settings.
:rtype: :obj:`argparse.Namespace`
"""
for _m in modules:
# Add configs to kuha_common_modules
configurable_modules[_m][0]()
settings.load_cli_args()
for _m in modules:
# Load cli args for kuha_common modules
configurable_modules[_m][1]()
return settings.get()
[docs]def get_settings():
"""Get loaded settings stored in :obj:`Settings`.
:returns: Loaded settings.
:rtype: :obj:`argparse.Namespace`
"""
return settings.get()
[docs]def add(*args, **kwargs):
r"""Module level function to add items to be
parsed in :obj:`Settings` singleton.
:param \*args: arguments passed to :meth:`Settings.add`.
:param \*\*kwargs: keyword arguments passed to :meth:`Settings.add`
"""
settings.add(*args, **kwargs)
[docs]def load(abs_dir_path, **kwargs):
r"""Module level function to load parser to :obj:`Settings` singleton.
:param abs_dir_path: absolute path to directory of the kuha application to be configured.
:type abs_dir_path: str
:param \*\*kwargs: keyword arguments passed to :meth:`Settings.load_parser`.
"""
settings.set_abs_dir_path(abs_dir_path)
settings.load_parser(**kwargs)
[docs]def prepend_abs_dir_path(path):
"""Helper function to prepend the stored absolute directory path to given argument.
:param path: end of the path.
:returns: absolute path ending to ``path``.
:rtype: str
"""
return '{}/{}'.format(settings.get_abs_dir_path(), path)