Backups/multi-user
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refacor: code formatting

Browse Source
This commit is contained in:
Swann 2019-07-23 20:18:51 +02:00
parent ea645044c6
commit 7ff01273be
3 changed files with 116 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
replication.py
View File

@ -1,19 +1,14 @@
import logging
from uuid import uuid4
import json
try:
from .libs import umsgpack
except:
# Server import
from libs import umsgpack
import zmq
import logging
import pickle
from enum import Enum
from uuid import uuid4
import zmq
logger = logging.getLogger(__name__)
class RepState(Enum):
ADDED = 0
COMMITED = 1
@ -32,7 +27,6 @@ class ReplicatedDataFactory(object):
self.register_type(str, RepCommand)
self.register_type(RepDeleteCommand, RepDeleteCommand)
def register_type(self, dtype, implementation):
"""
Register a new replicated datatype implementation
@ -127,7 +121,6 @@ class ReplicatedDatablock(object):
return instance
def store(self, dict, persistent=False):
"""
I want to store my replicated data. Persistent means into the disk
@ -142,21 +135,18 @@ class ReplicatedDatablock(object):
return self.uuid
def deserialize(self, data):
"""
BUFFER -> JSON
"""
raise NotImplementedError
def serialize(self, data):
"""
JSON -> BUFFER
"""
raise NotImplementedError
def dump(self):
"""
DCC -> JSON
@ -165,14 +155,12 @@ class ReplicatedDatablock(object):
return json.dumps(self.pointer)
def load(self, target=None):
"""
JSON -> DCC
"""
raise NotImplementedError
def resolve(self):
"""
I want to resolve my orphan data to an existing one
@ -181,7 +169,6 @@ class ReplicatedDatablock(object):
"""
raise NotImplementedError
def __repr__(self):
return "{uuid} - owner: {owner} - type: {type}".format(
uuid=self.uuid,
@ -189,6 +176,7 @@ class ReplicatedDatablock(object):
type=self.str_type
)
class RepCommand(ReplicatedDatablock):
def serialize(self, data):
return pickle.dumps(data)
@ -199,6 +187,7 @@ class RepCommand(ReplicatedDatablock):
def load(self, target):
target = self.pointer
class RepDeleteCommand(ReplicatedDatablock):
def serialize(self, data):
return pickle.dumps(data)
@ -213,7 +202,6 @@ class RepDeleteCommand(ReplicatedDatablock):
del rep_store[self.buffer]
# class RepObject(ReplicatedDatablock):
# def deserialize(self):
# try:

37
replication_client.py
View File

@ -1,8 +1,10 @@
import threading
import logging
import zmq
import threading
import time
from replication import ReplicatedDatablock, RepCommand,RepDeleteCommand
import zmq
from replication import RepCommand, RepDeleteCommand, ReplicatedDatablock
from replication_graph import ReplicationGraph
logger = logging.getLogger(__name__)
@ -54,7 +56,8 @@ class Client(object):
assert(object)
# Construct the coresponding replication type
new_item = self._factory.construct_from_dcc(object)(owner="client", pointer=object)
new_item = self._factory.construct_from_dcc(
object)(owner="client", pointer=object)
if new_item:
logger.info("Registering {} on {}".format(object, new_item.uuid))
@ -76,7 +79,9 @@ class Client(object):
"""
if object_uuid in self._rep_store.keys():
delete_command = RepDeleteCommand(owner='client', buffer=object_uuid)
delete_command = RepDeleteCommand(
owner='client', buffer=object_uuid)
# remove the key from our store
delete_command.store(self._rep_store)
delete_command.push(self._net_client.publish)
else:
@ -89,6 +94,7 @@ class Client(object):
"""
pass
class ClientNetService(threading.Thread):
def __init__(self, store_reference=None, factory=None, id="default"):
@ -147,7 +153,6 @@ class ClientNetService(threading.Thread):
self.snapshot.send(b"SNAPSHOT_REQUEST")
self.state = STATE_SYNCING
"""NET IN
Given the net state we do something:
SYNCING : load snapshots
@ -157,7 +162,8 @@ class ClientNetService(threading.Thread):
if self.snapshot in items:
if self.state == STATE_SYNCING:
datablock = ReplicatedDatablock.pull(self.snapshot, self._factory)
datablock = ReplicatedDatablock.pull(
self.snapshot, self._factory)
if 'SNAPSHOT_END' in datablock.buffer:
self.state = STATE_ACTIVE
@ -168,14 +174,15 @@ class ClientNetService(threading.Thread):
# We receive updates from the server !
if self.subscriber in items:
if self.state == STATE_ACTIVE:
logger.debug("{} : Receiving changes from server".format(self._id))
datablock = ReplicatedDatablock.pull(self.subscriber, self._factory)
logger.debug(
"{} : Receiving changes from server".format(self._id))
datablock = ReplicatedDatablock.pull(
self.subscriber, self._factory)
datablock.store(self._store_reference)
if not items:
logger.error("No request ")
self.snapshot.close()
self.subscriber.close()
self.publish.close()
@ -192,11 +199,11 @@ class ClientNetService(threading.Thread):
self.state = 0
class Server():
def __init__(self, config=None, factory=None):
self._rep_store = {}
self._net = ServerNetService(store_reference=self._rep_store, factory=factory)
self._net = ServerNetService(
store_reference=self._rep_store, factory=factory)
def serve(self, port=5560):
self._net.listen(port=port)
@ -227,7 +234,6 @@ class ServerNetService(threading.Thread):
self.factory = factory
self.clients = {}
def listen(self, port=5560):
try:
# Update request
@ -288,8 +294,8 @@ class ServerNetService(threading.Thread):
# Snapshot end
self.snapshot.send(identity, zmq.SNDMORE)
RepCommand(owner='server',pointer='SNAPSHOT_END').push(self.snapshot)
RepCommand(owner='server', pointer='SNAPSHOT_END').push(
self.snapshot)
# Regular update routing (Clients / Server / Clients)
if self.pull in socks:
@ -312,7 +318,6 @@ class ServerNetService(threading.Thread):
self._exit_event.clear()
def stop(self):
self._exit_event.set()

29
test_replication.py
View File

@ -1,17 +1,20 @@
import unittest
from replication import ReplicatedDatablock, ReplicatedDataFactory
import umsgpack
import logging
from replication_client import Client, Server
import time
import cProfile
import logging
import re
import time
import unittest
logging.basicConfig()
logging.getLogger().setLevel(logging.INFO)
logger = logging.getLogger(__name__)
from replication import ReplicatedDatablock, ReplicatedDataFactory
from replication_client import Client, Server
class SampleData():
def __init__(self, map={"sample": "data"}):
self.map = map
@ -28,14 +31,12 @@ class RepSampleData(ReplicatedDatablock):
return pickle.loads(data)
def dump(self):
import json
output = {}
output['map'] = json.dumps(self.pointer.map)
return output
def load(self, target=None):
import json
if target is None:
@ -49,7 +50,8 @@ class TestDataFactory(unittest.TestCase):
factory = ReplicatedDataFactory()
factory.register_type(SampleData, RepSampleData)
data_sample = SampleData()
rep_sample = factory.construct_from_dcc(data_sample)(owner="toto", pointer=data_sample)
rep_sample = factory.construct_from_dcc(
data_sample)(owner="toto", pointer=data_sample)
self.assertEqual(isinstance(rep_sample, RepSampleData), True)
@ -91,7 +93,6 @@ class TestClient(unittest.TestCase):
# Test the key registering
data_sample_key = client.register(SampleData())
client2.connect(port=5575)
time.sleep(0.2)
rep_test_key = client2._rep_store[data_sample_key].uuid
@ -117,7 +118,6 @@ class TestClient(unittest.TestCase):
client2 = Client(factory=factory, id="cli2_test_register_client_data")
client2.connect(port=5560)
# Test the key registering
data_sample_key = client.register(SampleData())
@ -125,12 +125,10 @@ class TestClient(unittest.TestCase):
# Waiting for server to receive the datas
rep_test_key = client2._rep_store[data_sample_key].uuid
client.disconnect()
client2.disconnect()
server.stop()
self.assertEqual(rep_test_key, data_sample_key)
def test_client_data_intergity(self):
@ -157,12 +155,10 @@ class TestClient(unittest.TestCase):
client2._rep_store[data_sample_key].load(target=test_map_result)
client.disconnect()
client2.disconnect()
server.stop()
self.assertEqual(test_map_result.map["toto"], test_map["toto"])
def test_client_unregister_key(self):
@ -244,6 +240,7 @@ class TestStressClient(unittest.TestCase):
self.assertLess(total_time, 1)
def suite():
suite = unittest.TestSuite()
@ -261,7 +258,7 @@ def suite():
return suite
if __name__ == '__main__':
runner = unittest.TextTestRunner(verbosity=2)
runner.run(suite())