diff options
| -rwxr-xr-x | bin/send_msg | 2 | ||||
| -rw-r--r-- | eventmq/__init__.py | 2 | ||||
| -rw-r--r-- | eventmq/jobmanager.py | 24 | ||||
| -rw-r--r-- | eventmq/log.py | 6 | ||||
| -rw-r--r-- | setup.py | 2 |
5 files changed, 19 insertions, 17 deletions
diff --git a/bin/send_msg b/bin/send_msg index 33cfb1a..c3e88fa 100755 --- a/bin/send_msg +++ b/bin/send_msg | |||
| @@ -22,7 +22,7 @@ if __name__ == "__main__": | |||
| 22 | 'callable': 'work_job', | 22 | 'callable': 'work_job', |
| 23 | 'class_args': ('blurp',), | 23 | 'class_args': ('blurp',), |
| 24 | 'class_kwargs': {'kwarg1': True}, | 24 | 'class_kwargs': {'kwarg1': True}, |
| 25 | 'args': (1, ), | 25 | 'args': (10, ), |
| 26 | 'kwargs': {} | 26 | 'kwargs': {} |
| 27 | }] | 27 | }] |
| 28 | 28 | ||
diff --git a/eventmq/__init__.py b/eventmq/__init__.py index a02a091..ef87218 100644 --- a/eventmq/__init__.py +++ b/eventmq/__init__.py | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | __author__ = 'EventMQ Contributors' | 1 | __author__ = 'EventMQ Contributors' |
| 2 | __version__ = '0.3.4.9' | 2 | __version__ = '0.3.4.11' |
| 3 | 3 | ||
| 4 | PROTOCOL_VERSION = 'eMQP/1.0' | 4 | PROTOCOL_VERSION = 'eMQP/1.0' |
| 5 | 5 | ||
diff --git a/eventmq/jobmanager.py b/eventmq/jobmanager.py index c900495..78a080b 100644 --- a/eventmq/jobmanager.py +++ b/eventmq/jobmanager.py | |||
| @@ -20,7 +20,7 @@ Ensures things about jobs and spawns the actual tasks | |||
| 20 | from json import dumps as serializer, loads as deserializer | 20 | from json import dumps as serializer, loads as deserializer |
| 21 | 21 | ||
| 22 | import logging | 22 | import logging |
| 23 | from multiprocessing import Queue as mp_queue | 23 | from multiprocessing import Manager as MPManager |
| 24 | import os | 24 | import os |
| 25 | import signal | 25 | import signal |
| 26 | import sys | 26 | import sys |
| @@ -121,8 +121,9 @@ class JobManager(HeartbeatMixin, EMQPService): | |||
| 121 | self.pid_distribution = {} | 121 | self.pid_distribution = {} |
| 122 | 122 | ||
| 123 | #: Setup worker queues | 123 | #: Setup worker queues |
| 124 | self.request_queue = mp_queue() | 124 | self._mp_manager = MPManager() |
| 125 | self.finished_queue = mp_queue() | 125 | self.request_queue = self._mp_manager.Queue() |
| 126 | self.finished_queue = self._mp_manager.Queue() | ||
| 126 | self._setup() | 127 | self._setup() |
| 127 | 128 | ||
| 128 | def handle_pdb(self, sig, frame): | 129 | def handle_pdb(self, sig, frame): |
| @@ -146,6 +147,14 @@ class JobManager(HeartbeatMixin, EMQPService): | |||
| 146 | Starts the actual event loop. Usually called by :meth:`start` | 147 | Starts the actual event loop. Usually called by :meth:`start` |
| 147 | """ | 148 | """ |
| 148 | # Acknowledgment has come | 149 | # Acknowledgment has come |
| 150 | # When the job manager unexpectedly disconnects from the router and | ||
| 151 | # reconnects it needs to send a ready for each previously available | ||
| 152 | # worker. | ||
| 153 | # Send a READY for each previously available worker | ||
| 154 | if hasattr(self, '_workers'): | ||
| 155 | for _ in self._workers: | ||
| 156 | self.send_ready() | ||
| 157 | |||
| 149 | self.status = STATUS.running | 158 | self.status = STATUS.running |
| 150 | 159 | ||
| 151 | try: | 160 | try: |
| @@ -210,15 +219,6 @@ class JobManager(HeartbeatMixin, EMQPService): | |||
| 210 | except Exception: | 219 | except Exception: |
| 211 | logger.exception("Unhandled exception in main jobmanager loop") | 220 | logger.exception("Unhandled exception in main jobmanager loop") |
| 212 | 221 | ||
| 213 | # Cleanup | ||
| 214 | if hasattr(self, '_workers'): | ||
| 215 | del self._workers | ||
| 216 | |||
| 217 | # Flush the queues with workers | ||
| 218 | self.request_queue = mp_queue() | ||
| 219 | self.finished_queue = mp_queue() | ||
| 220 | logger.info("Reached end of event loop") | ||
| 221 | |||
| 222 | def handle_response(self, resp): | 222 | def handle_response(self, resp): |
| 223 | """ | 223 | """ |
| 224 | Handles a response from a worker process to the jobmanager | 224 | Handles a response from a worker process to the jobmanager |
diff --git a/eventmq/log.py b/eventmq/log.py index 1864c8d..3d62165 100644 --- a/eventmq/log.py +++ b/eventmq/log.py | |||
| @@ -27,9 +27,11 @@ import zmq.log.handlers | |||
| 27 | 27 | ||
| 28 | 28 | ||
| 29 | FORMAT_STANDARD = logging.Formatter( | 29 | FORMAT_STANDARD = logging.Formatter( |
| 30 | '%(asctime)s - %(name)s %(levelname)s - %(message)s') | 30 | '%(asctime)s - %(name)s %(levelname)s - %(message)s', |
| 31 | datefmt='%Y-%m-%dT%H:%M:%S%z') | ||
| 31 | FORMAT_NAMELESS = logging.Formatter( | 32 | FORMAT_NAMELESS = logging.Formatter( |
| 32 | '%(asctime)s - %(levelname)s - %(message)s') | 33 | '%(asctime)s - %(levelname)s - %(message)s', |
| 34 | datefmt='%Y-%m-%dT%H:%M:%S%z') | ||
| 33 | 35 | ||
| 34 | 36 | ||
| 35 | class PUBHandler(zmq.log.handlers.PUBHandler): | 37 | class PUBHandler(zmq.log.handlers.PUBHandler): |
| @@ -7,7 +7,7 @@ from setuptools import find_packages, setup | |||
| 7 | 7 | ||
| 8 | setup( | 8 | setup( |
| 9 | name='eventmq', | 9 | name='eventmq', |
| 10 | version='0.3.4.9', | 10 | version='0.3.4.11', |
| 11 | description='EventMQ job execution and messaging system based on ZeroMQ', | 11 | description='EventMQ job execution and messaging system based on ZeroMQ', |
| 12 | packages=find_packages(), | 12 | packages=find_packages(), |
| 13 | install_requires=['pyzmq==15.4.0', | 13 | install_requires=['pyzmq==15.4.0', |