aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authortpavelchak2020-02-07 15:43:26 +0200
committertpavelchak2020-02-07 15:43:26 +0200
commit962903afc7634d52444ec047a4ee59d9e19ec330 (patch)
treec02d0eee6667fb7a956d4a2fc0e32e67b1d70fdf
parent04702253c51579c883c93bc72e4f94c8f258f434 (diff)
downloadeventmq-962903afc7634d52444ec047a4ee59d9e19ec330.tar.gz
eventmq-962903afc7634d52444ec047a4ee59d9e19ec330.zip
Fix Python 2/3 compatibility issues
- version bump 0.3.12 - upgrade `six, flake8` libs - fix `flake8` check issues - add import of `from __future__ import print_function` - fix params for `zsocket.setsockopt()` and `hashlib.sha1()` methods - use `range, map` functions from `six` - wrap result of `map, dict.keys(), dict.values()` functions in a list - fix `EMQdeque.__unicode__()` method for py2 support - fix using of exception messages - update integer division for support the same behavior in py3 as is in py2 - refactor using of `filter` function
-rw-r--r--bin/logwatcher.py4
-rw-r--r--eventmq/__init__.py2
-rw-r--r--eventmq/jobmanager.py6
-rw-r--r--eventmq/pub.py1
-rw-r--r--eventmq/publisher.py4
-rw-r--r--eventmq/receiver.py4
-rw-r--r--eventmq/router.py12
-rw-r--r--eventmq/scheduler.py3
-rw-r--r--eventmq/subscriber.py5
-rw-r--r--eventmq/switch.py2
-rw-r--r--eventmq/tests/test_router.py3
-rw-r--r--eventmq/tests/test_utils.py1
-rw-r--r--eventmq/utils/__init__.py1
-rw-r--r--eventmq/utils/classes.py8
-rw-r--r--eventmq/utils/encoding.py1
-rw-r--r--eventmq/utils/functions.py4
-rw-r--r--eventmq/utils/messages.py4
-rw-r--r--eventmq/utils/settings.py2
-rw-r--r--eventmq/worker.py2
-rw-r--r--setup.py8
20 files changed, 48 insertions, 29 deletions
diff --git a/bin/logwatcher.py b/bin/logwatcher.py
index 1abeb4a..d76b0a1 100644
--- a/bin/logwatcher.py
+++ b/bin/logwatcher.py
@@ -1,9 +1,11 @@
1#!/usr/bin/env python 1#!/usr/bin/env python
2 2
3from __future__ import print_function
4
3import zmq 5import zmq
4 6
5s = zmq.Context.instance().socket(zmq.SUB) 7s = zmq.Context.instance().socket(zmq.SUB)
6s.setsockopt(zmq.SUBSCRIBE, '') 8s.setsockopt(zmq.SUBSCRIBE, b'')
7s.connect('tcp://127.0.0.1:33445') 9s.connect('tcp://127.0.0.1:33445')
8 10
9poller = zmq.Poller() 11poller = zmq.Poller()
diff --git a/eventmq/__init__.py b/eventmq/__init__.py
index 77bbe4a..d9b9226 100644
--- a/eventmq/__init__.py
+++ b/eventmq/__init__.py
@@ -1,5 +1,5 @@
1__author__ = 'EventMQ Contributors' 1__author__ = 'EventMQ Contributors'
2__version__ = '0.3.11' 2__version__ = '0.3.12'
3 3
4PROTOCOL_VERSION = 'eMQP/1.0' 4PROTOCOL_VERSION = 'eMQP/1.0'
5 5
diff --git a/eventmq/jobmanager.py b/eventmq/jobmanager.py
index f4a928c..e15e49f 100644
--- a/eventmq/jobmanager.py
+++ b/eventmq/jobmanager.py
@@ -18,7 +18,6 @@
18Ensures things about jobs and spawns the actual tasks 18Ensures things about jobs and spawns the actual tasks
19""" 19"""
20from json import dumps as serializer, loads as deserializer 20from json import dumps as serializer, loads as deserializer
21
22import logging 21import logging
23from multiprocessing import Manager as MPManager 22from multiprocessing import Manager as MPManager
24import os 23import os
@@ -26,6 +25,7 @@ import signal
26import sys 25import sys
27import time 26import time
28 27
28from six.moves import range
29import zmq 29import zmq
30 30
31from eventmq.log import setup_logger 31from eventmq.log import setup_logger
@@ -142,7 +142,7 @@ class JobManager(HeartbeatMixin, EMQPService):
142 w.start() 142 w.start()
143 self._workers[w.pid] = w 143 self._workers[w.pid] = w
144 144
145 return self._workers.values() 145 return list(self._workers.values())
146 146
147 def _start_event_loop(self): 147 def _start_event_loop(self):
148 """ 148 """
@@ -333,7 +333,7 @@ class JobManager(HeartbeatMixin, EMQPService):
333 Worker died of natural causes, ensure its death and 333 Worker died of natural causes, ensure its death and
334 remove from tracking, will be replaced on next heartbeat 334 remove from tracking, will be replaced on next heartbeat
335 """ 335 """
336 if pid in self._workers.keys(): 336 if pid in list(self._workers.keys()):
337 del self._workers[pid] 337 del self._workers[pid]
338 338
339 def worker_ready(self, reply, msgid, death, pid): 339 def worker_ready(self, reply, msgid, death, pid):
diff --git a/eventmq/pub.py b/eventmq/pub.py
index 3918580..8575278 100644
--- a/eventmq/pub.py
+++ b/eventmq/pub.py
@@ -20,7 +20,6 @@ Publishes messages to subscribers
20import logging 20import logging
21 21
22from eventmq.log import setup_logger 22from eventmq.log import setup_logger
23
24from . import conf, poller, publisher, receiver 23from . import conf, poller, publisher, receiver
25from .constants import STATUS 24from .constants import STATUS
26from .utils.classes import HeartbeatMixin 25from .utils.classes import HeartbeatMixin
diff --git a/eventmq/publisher.py b/eventmq/publisher.py
index 2e77bcd..cf29a2e 100644
--- a/eventmq/publisher.py
+++ b/eventmq/publisher.py
@@ -19,6 +19,7 @@ Publishes messages to subscribers
19""" 19"""
20import logging 20import logging
21 21
22import six
22import zmq 23import zmq
23 24
24from . import constants 25from . import constants
@@ -36,7 +37,8 @@ class Publisher():
36 37
37 def __init__(self, *args, **kwargs): 38 def __init__(self, *args, **kwargs):
38 self.zcontext = kwargs.get('context', zmq.Context.instance()) 39 self.zcontext = kwargs.get('context', zmq.Context.instance())
39 self.name = kwargs.get('name', generate_device_name()) 40 self.name = six.ensure_binary(kwargs.get('name',
41 generate_device_name()))
40 42
41 self.zsocket = kwargs.get('socket', self.zcontext.socket(zmq.PUB)) 43 self.zsocket = kwargs.get('socket', self.zcontext.socket(zmq.PUB))
42 self.zsocket.setsockopt(zmq.IDENTITY, self.name) 44 self.zsocket.setsockopt(zmq.IDENTITY, self.name)
diff --git a/eventmq/receiver.py b/eventmq/receiver.py
index dc0ead8..b901405 100644
--- a/eventmq/receiver.py
+++ b/eventmq/receiver.py
@@ -19,6 +19,7 @@ The receiver is responsible for receiveing messages
19""" 19"""
20import logging 20import logging
21 21
22import six
22import zmq 23import zmq
23 24
24from . import conf, constants 25from . import conf, constants
@@ -59,7 +60,8 @@ class Receiver(ZMQReceiveMixin, ZMQSendMixin):
59 self.zcontext = kwargs.get('context', zmq.Context.instance()) 60 self.zcontext = kwargs.get('context', zmq.Context.instance())
60 self.zcontext.set(zmq.MAX_SOCKETS, conf.MAX_SOCKETS) 61 self.zcontext.set(zmq.MAX_SOCKETS, conf.MAX_SOCKETS)
61 62
62 self.name = kwargs.get('name', generate_device_name()) 63 self.name = six.ensure_binary(kwargs.get('name',
64 generate_device_name()))
63 65
64 self.zsocket = kwargs.get('socket', self.zcontext.socket(zmq.ROUTER)) 66 self.zsocket = kwargs.get('socket', self.zcontext.socket(zmq.ROUTER))
65 self.zsocket.setsockopt(zmq.IDENTITY, self.name) 67 self.zsocket.setsockopt(zmq.IDENTITY, self.name)
diff --git a/eventmq/router.py b/eventmq/router.py
index 81ca257..3c141c3 100644
--- a/eventmq/router.py
+++ b/eventmq/router.py
@@ -22,6 +22,8 @@ import json # deserialize queues in on_inform. should be refactored
22import logging 22import logging
23import signal 23import signal
24 24
25from six.moves import map
26
25from eventmq.log import setup_logger, setup_wal_logger 27from eventmq.log import setup_logger, setup_wal_logger
26from . import __version__ 28from . import __version__
27from . import conf, constants, exceptions, poller, receiver 29from . import conf, constants, exceptions, poller, receiver
@@ -407,7 +409,7 @@ class Router(HeartbeatMixin):
407 # Assumes the highest priority queue comes first 409 # Assumes the highest priority queue comes first
408 for queue in queue_names: 410 for queue in queue_names:
409 queue_name = queue[1] 411 queue_name = queue[1]
410 if queue_name in self.waiting_messages.keys(): 412 if queue_name in list(self.waiting_messages.keys()):
411 logger.debug('Found waiting message in the %s waiting_messages' 413 logger.debug('Found waiting message in the %s waiting_messages'
412 ' queue' % queue_name) 414 ' queue' % queue_name)
413 msg = self.waiting_messages[queue_name].peekleft() 415 msg = self.waiting_messages[queue_name].peekleft()
@@ -476,7 +478,7 @@ class Router(HeartbeatMixin):
476 total_mem = psutil.virtual_memory().total 478 total_mem = psutil.virtual_memory().total
477 # Set queue limit to be 75% of total memory with ~100 byte 479 # Set queue limit to be 75% of total memory with ~100 byte
478 # messages 480 # messages
479 limit = int((total_mem / 100) * 0.75) 481 limit = int(int(total_mem / 100) * 0.75)
480 self.waiting_messages[queue_name] = EMQdeque( 482 self.waiting_messages[queue_name] = EMQdeque(
481 full=limit, on_full=router_on_full) 483 full=limit, on_full=router_on_full)
482 else: 484 else:
@@ -866,7 +868,7 @@ class Router(HeartbeatMixin):
866 for queue in worker['queues']: 868 for queue in worker['queues']:
867 name = queue[1] 869 name = queue[1]
868 workers = self.queues[name] 870 workers = self.queues[name]
869 revised_list = filter(lambda x: x[1] != worker_id, workers) 871 revised_list = [x for x in workers if x[1] != worker_id]
870 self.queues[name] = revised_list 872 self.queues[name] = revised_list
871 logger.debug('Removed worker - {} from {}'.format(worker_id, name)) 873 logger.debug('Removed worker - {} from {}'.format(worker_id, name))
872 874
@@ -879,8 +881,8 @@ class Router(HeartbeatMixin):
879 """ 881 """
880 self.schedulers.pop(scheduler_id) 882 self.schedulers.pop(scheduler_id)
881 schedulers_to_remove = self.scheduler_queue 883 schedulers_to_remove = self.scheduler_queue
882 self.scheduler_queue = filter(lambda x: x != scheduler_id, 884 self.scheduler_queue = \
883 schedulers_to_remove) 885 [x for x in schedulers_to_remove if x != scheduler_id]
884 logger.debug('Removed scheduler - {} from known schedulers'.format( 886 logger.debug('Removed scheduler - {} from known schedulers'.format(
885 scheduler_id)) 887 scheduler_id))
886 888
diff --git a/eventmq/scheduler.py b/eventmq/scheduler.py
index e7c13ec..afdb116 100644
--- a/eventmq/scheduler.py
+++ b/eventmq/scheduler.py
@@ -17,6 +17,8 @@
17============================= 17=============================
18Handles cron and other scheduled tasks 18Handles cron and other scheduled tasks
19""" 19"""
20from __future__ import print_function
21
20from hashlib import sha1 as emq_hash 22from hashlib import sha1 as emq_hash
21import importlib 23import importlib
22import json 24import json
@@ -29,7 +31,6 @@ from croniter import croniter
29from six import iteritems, next 31from six import iteritems, next
30 32
31from eventmq.log import setup_logger 33from eventmq.log import setup_logger
32
33from . import __version__ 34from . import __version__
34from . import conf, constants 35from . import conf, constants
35from .client.messages import send_request 36from .client.messages import send_request
diff --git a/eventmq/subscriber.py b/eventmq/subscriber.py
index ef1aa0b..5aef82f 100644
--- a/eventmq/subscriber.py
+++ b/eventmq/subscriber.py
@@ -1,7 +1,10 @@
1""" 1"""
2derp subscriber 2derp subscriber
3""" 3"""
4from __future__ import print_function
5
4from past.builtins import xrange 6from past.builtins import xrange
7import six
5import zmq 8import zmq
6 9
7 10
@@ -11,7 +14,7 @@ if __name__ == "__main__":
11 ctx = zmq.Context() 14 ctx = zmq.Context()
12 s = ctx.socket(zmq.SUB) 15 s = ctx.socket(zmq.SUB)
13 s.linger = 0 16 s.linger = 0
14 s.setsockopt(zmq.SUBSCRIBE, str(i)) 17 s.setsockopt(zmq.SUBSCRIBE, six.ensure_binary(str(i)))
15 s.connect('tcp://127.0.0.1:47299') 18 s.connect('tcp://127.0.0.1:47299')
16 sockets.append(s) 19 sockets.append(s)
17 20
diff --git a/eventmq/switch.py b/eventmq/switch.py
index f6e94b4..5fe6861 100644
--- a/eventmq/switch.py
+++ b/eventmq/switch.py
@@ -5,5 +5,5 @@ if __name__ == "__main__":
5 # switch.listen('tcp://127.0.0.1:47331', 'tcp://127.0.0.1:47330') 5 # switch.listen('tcp://127.0.0.1:47331', 'tcp://127.0.0.1:47330')
6 try: 6 try:
7 switch.start() 7 switch.start()
8 except: 8 except Exception:
9 switch.stop() 9 switch.stop()
diff --git a/eventmq/tests/test_router.py b/eventmq/tests/test_router.py
index cf7c9ef..0a315e3 100644
--- a/eventmq/tests/test_router.py
+++ b/eventmq/tests/test_router.py
@@ -18,6 +18,7 @@ import unittest
18 18
19from freezegun import freeze_time 19from freezegun import freeze_time
20import mock 20import mock
21from six.moves import range
21from testfixtures import LogCapture 22from testfixtures import LogCapture
22import zmq 23import zmq
23 24
@@ -318,7 +319,7 @@ class TestCase(unittest.TestCase):
318 319
319 # There should be no keys because the code checks for their existence 320 # There should be no keys because the code checks for their existence
320 # to know if there is a waiting message 321 # to know if there is a waiting message
321 self.assertEqual(0, len(self.router.waiting_messages.keys())) 322 self.assertEqual(0, len(list(self.router.waiting_messages.keys())))
322 323
323 # No waiting messages 324 # No waiting messages
324 self.router.on_ready(worker1_id, ready_msgid1, ['READY', ready_msgid1]) 325 self.router.on_ready(worker1_id, ready_msgid1, ['READY', ready_msgid1])
diff --git a/eventmq/tests/test_utils.py b/eventmq/tests/test_utils.py
index 5173832..e71f2e3 100644
--- a/eventmq/tests/test_utils.py
+++ b/eventmq/tests/test_utils.py
@@ -30,6 +30,7 @@ import sys
30import unittest 30import unittest
31 31
32import mock 32import mock
33from six.moves import range
33 34
34from .. import conf 35from .. import conf
35from .. import constants 36from .. import constants
diff --git a/eventmq/utils/__init__.py b/eventmq/utils/__init__.py
index dad8299..1f65d65 100644
--- a/eventmq/utils/__init__.py
+++ b/eventmq/utils/__init__.py
@@ -27,6 +27,7 @@ like creating message more simple.
27 settings 27 settings
28 timeutils 28 timeutils
29""" 29"""
30from six.moves import map
30 31
31 32
32def random_characters(): 33def random_characters():
diff --git a/eventmq/utils/classes.py b/eventmq/utils/classes.py
index 5788e94..14409ea 100644
--- a/eventmq/utils/classes.py
+++ b/eventmq/utils/classes.py
@@ -422,7 +422,7 @@ class ZMQSendMixin(object):
422 self.zsocket.send_multipart(msg, 422 self.zsocket.send_multipart(msg,
423 flags=zmq.NOBLOCK) 423 flags=zmq.NOBLOCK)
424 except zmq.error.ZMQError as e: 424 except zmq.error.ZMQError as e:
425 if 'No route' in e.message: 425 if 'No route' in str(e):
426 raise exceptions.PeerGoneAwayError(e) 426 raise exceptions.PeerGoneAwayError(e)
427 427
428 def send(self, message, protocol_version): 428 def send(self, message, protocol_version):
@@ -469,7 +469,7 @@ class EMQdeque(object):
469 return "{}".format(str(self._queue)) 469 return "{}".format(str(self._queue))
470 470
471 def __unicode__(self): 471 def __unicode__(self):
472 return "{}".format(six.u(self._queue)) 472 return "{}".format(six.text_type(self._queue))
473 473
474 def __repr__(self): 474 def __repr__(self):
475 return "{}".format(repr(self._queue)) 475 return "{}".format(repr(self._queue))
@@ -519,7 +519,7 @@ class EMQdeque(object):
519 bool: True if the deque contains at least ``full`` items. False 519 bool: True if the deque contains at least ``full`` items. False
520 otherwise 520 otherwise
521 """ 521 """
522 if self.full and self.full is not 0: 522 if self.full and self.full != 0:
523 return len(self._queue) >= self.full 523 return len(self._queue) >= self.full
524 else: 524 else:
525 return False 525 return False
@@ -541,7 +541,7 @@ class EMQdeque(object):
541 bool: True if the deque contains at least ``pfull`` items. 541 bool: True if the deque contains at least ``pfull`` items.
542 False otherwise 542 False otherwise
543 """ 543 """
544 if self.pfull and self.pfull is not 0: 544 if self.pfull and self.pfull != 0:
545 return len(self._queue) >= self.pfull 545 return len(self._queue) >= self.pfull
546 else: 546 else:
547 return False 547 return False
diff --git a/eventmq/utils/encoding.py b/eventmq/utils/encoding.py
index d45624e..333ff48 100644
--- a/eventmq/utils/encoding.py
+++ b/eventmq/utils/encoding.py
@@ -13,6 +13,7 @@
13# You should have received a copy of the GNU Lesser General Public License 13# You should have received a copy of the GNU Lesser General Public License
14# along with eventmq. If not, see <http://www.gnu.org/licenses/>. 14# along with eventmq. If not, see <http://www.gnu.org/licenses/>.
15from past.builtins import basestring 15from past.builtins import basestring
16from six.moves import range
16 17
17from .. import conf 18from .. import conf
18 19
diff --git a/eventmq/utils/functions.py b/eventmq/utils/functions.py
index f47526b..556107d 100644
--- a/eventmq/utils/functions.py
+++ b/eventmq/utils/functions.py
@@ -4,6 +4,8 @@ import inspect
4import json 4import json
5import sys 5import sys
6 6
7import six
8
7from .. import log 9from .. import log
8from ..exceptions import CallableFromPathError 10from ..exceptions import CallableFromPathError
9 11
@@ -70,7 +72,7 @@ def arguments_hash(*args, **kwargs):
70 } 72 }
71 73
72 data = json.dumps(args, cls=IgnoreJSONEncoder) 74 data = json.dumps(args, cls=IgnoreJSONEncoder)
73 return hashlib.sha1(data).hexdigest() 75 return hashlib.sha1(six.ensure_binary(data)).hexdigest()
74 76
75 77
76def name_from_callable(func): 78def name_from_callable(func):
diff --git a/eventmq/utils/messages.py b/eventmq/utils/messages.py
index 02ac129..d652c4c 100644
--- a/eventmq/utils/messages.py
+++ b/eventmq/utils/messages.py
@@ -170,8 +170,8 @@ def fwd_emqp_router_message(socket, recipient_id, payload):
170 flags=zmq.NOBLOCK) 170 flags=zmq.NOBLOCK)
171 except zmq.error.ZMQError as e: 171 except zmq.error.ZMQError as e:
172 if e.errno in errnos: 172 if e.errno in errnos:
173 e.message = e.message + " {}".format(recipient_id) 173 e_message = str(e) + " {}".format(recipient_id)
174 raise exceptions.PeerGoneAwayError(e) 174 raise exceptions.PeerGoneAwayError(e_message)
175 else: 175 else:
176 raise exceptions.EventMQError("errno {}: {}".format(e.errno, 176 raise exceptions.EventMQError("errno {}: {}".format(e.errno,
177 str(e))) 177 str(e)))
diff --git a/eventmq/utils/settings.py b/eventmq/utils/settings.py
index ade76c2..700d983 100644
--- a/eventmq/utils/settings.py
+++ b/eventmq/utils/settings.py
@@ -28,6 +28,8 @@ import json
28import logging 28import logging
29import os 29import os
30 30
31from six.moves import map
32
31from . import tuplify 33from . import tuplify
32from .. import conf 34from .. import conf
33 35
diff --git a/eventmq/worker.py b/eventmq/worker.py
index 1f3a6fa..0b22bad 100644
--- a/eventmq/worker.py
+++ b/eventmq/worker.py
@@ -93,7 +93,7 @@ class MultiprocessWorker(Process):
93 if os.getppid() != self.ppid: 93 if os.getppid() != self.ppid:
94 break 94 break
95 continue 95 continue
96 except Exception as e: 96 except Exception:
97 break 97 break
98 finally: 98 finally:
99 # If I'm an orphan, die 99 # If I'm an orphan, die
diff --git a/setup.py b/setup.py
index b4adf1d..3bcbc4c 100644
--- a/setup.py
+++ b/setup.py
@@ -20,7 +20,7 @@ setup(
20 packages=find_packages(), 20 packages=find_packages(),
21 install_requires=[ 21 install_requires=[
22 'pyzmq==18.1.0', 22 'pyzmq==18.1.0',
23 'six==1.10.0', 23 'six==1.14.0',
24 'monotonic==0.4', 24 'monotonic==0.4',
25 'croniter==0.3.10', 25 'croniter==0.3.10',
26 'future==0.15.2', 26 'future==0.15.2',
@@ -29,9 +29,9 @@ setup(
29 extras_require={ 29 extras_require={
30 'docs': ['Sphinx==1.5.2', ], 30 'docs': ['Sphinx==1.5.2', ],
31 'testing': [ 31 'testing': [
32 'flake8==3.2.1', 32 'flake8==3.7.8',
33 'flake8-import-order==0.11', 33 'flake8-import-order==0.18.1',
34 'flake8-print==2.0.2', 34 'flake8-print==3.1.0',
35 'coverage==4.0.3', 35 'coverage==4.0.3',
36 'testfixtures==4.7.0', 36 'testfixtures==4.7.0',
37 'freezegun==0.3.7', 37 'freezegun==0.3.7',