From 74df38d78e8bb3393c39c73d286874752b85511b Mon Sep 17 00:00:00 2001 From: jason Date: Tue, 17 Jul 2018 15:45:16 -0600 Subject: Allow configuration options to be specified as environment variables --- docs/settings_file.rst | 12 ++++++-- etc/eventmq.conf-dist | 5 +++- eventmq/__init__.py | 2 +- eventmq/tests/test_router.py | 13 +++++---- eventmq/utils/settings.py | 68 ++++++++++++++++++++++++++++---------------- setup.py | 2 +- 6 files changed, 67 insertions(+), 35 deletions(-) diff --git a/docs/settings_file.rst b/docs/settings_file.rst index d09f124..04fe451 100644 --- a/docs/settings_file.rst +++ b/docs/settings_file.rst @@ -1,7 +1,13 @@ ############################## Server Settings (eventmq.conf) ############################## -EventMQ uses a standard INI style config file found at ``/etc/eventmq.conf``. +EventMQ uses a standard INI style config file with the default +location of ``/etc/eventmq.conf``. If you would like to specify a custom path +you can use the ``EVENTMQ_CONFIG_FILE`` environment variable. + +All of these options can be defined via environment variables by converting +them to upper case and prefixing them with ``EVENTMQ_``. For example +``EVENTMQ_MAX_SOCKETS=2048``. ****** Global @@ -37,7 +43,7 @@ Default: 'tcp://127.0.0.1:47291' The address used to listen for connections from workers wal -======= +=== Default: '/var/log/eventmq/wal.log' Write-ahead Log for replaying messages received by the Router. Will @@ -45,7 +51,7 @@ try to create the directory specified and append to the filename given. Requires correct permissions to write to the given file. wal_enabled -=============== +=========== Default: False Enable or disable the Write-ahead Log diff --git a/etc/eventmq.conf-dist b/etc/eventmq.conf-dist index 9a59f2a..96dbad4 100644 --- a/etc/eventmq.conf-dist +++ b/etc/eventmq.conf-dist @@ -2,6 +2,9 @@ # Enable message output at different stages in the app. super_debug = true +# Number of maximum sockets to open per context/process. +max_sockets = 1024 + # Hide the heartbeat logs when super_debug is enabled. Showing them will generate a lot of messages. hide_heartbeat_logs = True @@ -37,4 +40,4 @@ concurrent_jobs=2 [publisher] publisher_incoming_addr=tcp://0.0.0.0:47298 -publisher_outgoing_addr=tcp://0.0.0.0:47299 \ No newline at end of file +publisher_outgoing_addr=tcp://0.0.0.0:47299 diff --git a/eventmq/__init__.py b/eventmq/__init__.py index c3ffd05..ed64a39 100644 --- a/eventmq/__init__.py +++ b/eventmq/__init__.py @@ -1,5 +1,5 @@ __author__ = 'EventMQ Contributors' -__version__ = '0.3.7' +__version__ = '0.3.8' PROTOCOL_VERSION = 'eMQP/1.0' diff --git a/eventmq/tests/test_router.py b/eventmq/tests/test_router.py index e9efa25..cf7c9ef 100644 --- a/eventmq/tests/test_router.py +++ b/eventmq/tests/test_router.py @@ -12,6 +12,7 @@ # # You should have received a copy of the GNU Lesser General Public License # along with eventmq. If not, see . +from imp import reload import json import unittest @@ -35,6 +36,10 @@ class TestCase(unittest.TestCase): @mock.patch('eventmq.receiver.zmq.Socket.bind') @mock.patch('eventmq.router.Router._start_event_loop') def test_start(self, event_loop_mock, zsocket_bind_mock): + # set the config file back to defaults. prevents tests from failing + # when there is a config file on the filesystem + reload(conf) + # Test default args self.router.start() self.router.incoming.listen.assert_called_with(conf.FRONTEND_ADDR) @@ -671,11 +676,9 @@ class TestCase(unittest.TestCase): self.router.process_client_message( (s1, '', constants.PROTOCOL_VERSION, command, msgid) + msg) - self.assertNotIn(s1, self.router.scheduler_queue, - 'Scheduler not ' - 'removed. {' - '}'.format( - self.router.scheduler_queue)) + self.assertNotIn( + s1, self.router.scheduler_queue, + 'Scheduler not removed. {}'.format(self.router.scheduler_queue)) def test_handle_kbye_from_worker(self): msgid = 'msg10' diff --git a/eventmq/utils/settings.py b/eventmq/utils/settings.py index 30c4dee..9365bff 100644 --- a/eventmq/utils/settings.py +++ b/eventmq/utils/settings.py @@ -16,7 +16,7 @@ :mod:`settings` -- Settings Utilities ===================================== """ -from configparser import ConfigParser +from configparser import ConfigParser, NoOptionError import json import logging import os @@ -30,26 +30,51 @@ logger = logging.getLogger(__name__) def import_settings(section='global'): """ - Import settings and apply to configuration globals + Import settings and apply to configuration globals. This function will + also read from the environment variables and override any value defined + in the file. Args: section (str): Name of the INI section to import """ config = ConfigParser() - if os.path.exists(conf.CONFIG_FILE): - config.read(conf.CONFIG_FILE) + config_path = os.environ.get('EVENTMQ_CONFIG_FILE', conf.CONFIG_FILE) + use_config_file = False - if not config.has_section(section): + if os.path.exists(config_path): + config.read(config_path) + if config.has_section(section): + use_config_file = True + else: logger.warning( - 'Tried to read nonexistent section {}'.format(section)) - return + 'Tried to read nonexistent section {} from {}'.format( + section, config_path)) - for name, value in config.items(section): - if hasattr(conf, name.upper()): - default_value = getattr(conf, name.upper()) - t = type(default_value) - if isinstance(default_value, (list, tuple)): + for name in dir(conf): + if name.startswith('_'): + continue + + value = None + found_value = False + default_value = getattr(conf, name) + + # Favor environment variables over the config file definition + try: + value = os.environ['EVENTMQ_{}'.format(name)] + found_value = True + except KeyError: + if use_config_file: + try: + value = config.get(section, name) + found_value = True + except NoOptionError: + found_value = False + + if found_value: + t = type(getattr(conf, name)) + + if t in (list, tuple): try: value = t(json.loads(value)) except ValueError: @@ -60,19 +85,14 @@ def import_settings(section='global'): # convert those elements, otherwise whatever it's type is # correct if isinstance(default_value[0], tuple): - setattr(conf, name.upper(), - t(map(tuplify, value))) + setattr(conf, name, t(map(tuplify, value))) else: - setattr(conf, name.upper(), t(value)) + setattr(conf, name, t(value)) elif isinstance(default_value, bool): - setattr(conf, name.upper(), + setattr(conf, name, True if 't' in value.lower() else False) else: - setattr(conf, name.upper(), t(value)) - logger.debug("Setting conf.{} to {}".format( - name.upper(), getattr(conf, name.upper()))) - else: - logger.warning('Tried to set invalid setting: %s' % name) - else: - logger.warning('Config file at {} not found. Continuing with ' - 'defaults.'.format(conf.CONFIG_FILE)) + setattr(conf, name, t(value)) + + logger.debug("Setting conf.{} to {}".format( + name, getattr(conf, name))) diff --git a/setup.py b/setup.py index e053544..741a517 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ from setuptools import find_packages, setup setup( name='eventmq', - version='0.3.7', + version='0.3.8', description='EventMQ job execution and messaging system based on ZeroMQ', packages=find_packages(), install_requires=['pyzmq==15.4.0', -- cgit v1.2.1