From 8e30406b84bf2133f760a451098551e34e9aaaf2 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Fri, 28 Nov 2014 10:28:53 +0100 Subject: [PATCH 1/7] add redundant check for snapshot integrity --- client/client_daemon.py | 43 ++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/client/client_daemon.py b/client/client_daemon.py index ab3be5a..7977261 100644 --- a/client/client_daemon.py +++ b/client/client_daemon.py @@ -868,10 +868,20 @@ def save_snapshot(self, timestamp): f.write( json.dumps({"timestamp": float(timestamp), "snapshot": self.last_status['snapshot']})) + def _remove_snap_orphan(self, md5_file): + self.local_full_snapshot[md5_file] = list(set(self.local_full_snapshot[md5_file])) + for path in list(self.local_full_snapshot[md5_file]): + if not os.path.exists(path): + self.local_full_snapshot[md5_file].remove(path) + def update_snapshot_upload(self, body): """ update of local full snapshot by upload request""" - self.local_full_snapshot[self.file_snapMd5( - body['src_path'])] = [get_relpath(body["src_path"])] + md5_file = self.file_snapMd5(body['src_path']) + if md5_file in self.local_full_snapshot: + self.local_full_snapshot[md5_file].append(get_relpath(body["src_path"])) + self._remove_snap_orphan(md5_file) + else: + self.local_full_snapshot[md5_file] = [get_relpath(body["src_path"])] def update_snapshot_update(self, body): """ update of local full snapshot by update request""" @@ -882,6 +892,7 @@ def update_snapshot_update(self, body): # is a copy of another file self.local_full_snapshot[new_file_md5].append( get_relpath(body['src_path'])) + self._remove_snap_orphan(md5_file) else: # else create a new md5 self.local_full_snapshot[new_file_md5] = [get_relpath( @@ -889,24 +900,34 @@ def update_snapshot_update(self, body): def update_snapshot_copy(self, body): """ update of local full snapshot by copy request""" - self.local_full_snapshot[self.file_snapMd5( - body['src_path'])].append(get_relpath(body["dst_path"])) + md5_file = self.file_snapMd5(body['src_path']) + paths_of_file = self.local_full_snapshot[md5_file] + paths_of_file.append(get_relpath(body["dst_path"])) + self._remove_snap_orphan(md5_file) def update_snapshot_move(self, body): """ update of local full snapshot by move request""" - paths_of_file = self.local_full_snapshot[self.file_snapMd5( - get_abspath(body["dst_path"]))] - paths_of_file.remove(get_relpath(body["src_path"])) + md5_file = self.file_snapMd5(get_abspath(body["dst_path"])) + paths_of_file = self.local_full_snapshot[md5_file] + try: + paths_of_file.remove(get_relpath(body["src_path"])) + except ValueError: + pass paths_of_file.append(get_relpath(body["dst_path"])) + self._remove_snap_orphan(md5_file) def update_snapshot_delete(self, body): """ update of local full snapshot by delete request""" md5_file = self.find_file_md5(self.local_full_snapshot, get_relpath(body['src_path']), False) + paths_of_file = self.local_full_snapshot[md5_file] logger.debug("find md5: {}".format(md5_file)) - if len(self.local_full_snapshot[md5_file]) == 1: - del self.local_full_snapshot[md5_file] - else: - self.local_full_snapshot[md5_file].remove(get_relpath(body['src_path'])) + if md5_file in self.local_full_snapshot: + try: + paths_of_file.remove(get_relpath(body['src_path'])) + except ValueError: + pass + if len(self.local_full_snapshot[md5_file]) == 0: + del paths_of_file logger.debug("path deleted: " + get_relpath(body['src_path'])) def save_timestamp(self, timestamp): From 4c5a60f451324dc49b92b7dbf548bbd5890b8dcd Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Fri, 28 Nov 2014 11:14:23 +0100 Subject: [PATCH 2/7] fix refuse --- client/client_daemon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/client_daemon.py b/client/client_daemon.py index 7977261..b3b1297 100644 --- a/client/client_daemon.py +++ b/client/client_daemon.py @@ -892,7 +892,7 @@ def update_snapshot_update(self, body): # is a copy of another file self.local_full_snapshot[new_file_md5].append( get_relpath(body['src_path'])) - self._remove_snap_orphan(md5_file) + self._remove_snap_orphan(new_file_md5) else: # else create a new md5 self.local_full_snapshot[new_file_md5] = [get_relpath( @@ -927,7 +927,7 @@ def update_snapshot_delete(self, body): except ValueError: pass if len(self.local_full_snapshot[md5_file]) == 0: - del paths_of_file + del self.local_full_snapshot[md5_file] logger.debug("path deleted: " + get_relpath(body['src_path'])) def save_timestamp(self, timestamp): From 84c81ec0634212cdb5c5d2838714fabf081feb77 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Fri, 28 Nov 2014 11:16:51 +0100 Subject: [PATCH 3/7] inhibit _remove_snapshot_orphan function in unit test --- client/test_client_daemon.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/client/test_client_daemon.py b/client/test_client_daemon.py index 1ccef42..131d812 100644 --- a/client/test_client_daemon.py +++ b/client/test_client_daemon.py @@ -1182,6 +1182,9 @@ def test_update_snapshot_upload(self): self.assertEqual(self.snapshot_manager.local_full_snapshot, original_snapshot) def test_update_snapshot_update(self): + #inhibit _remove_snap_orphan function + self.snapshot_manager._remove_snap_orphan = self._void_function + original_snapshot = copy.deepcopy(self.snapshot_manager.local_full_snapshot) mock_snapshot = copy.deepcopy(original_snapshot) mock_file_content = "test_content" @@ -1226,7 +1229,13 @@ def test_update_snapshot_update(self): } self.snapshotAsserEqual(self.snapshot_manager.local_full_snapshot, mock_snapshot) + def _void_function(self, *args, **kwargs): + pass + def test_update_snapshot_copy(self): + #inhibit _remove_snap_orphan function + self.snapshot_manager._remove_snap_orphan = self._void_function + mock_snapshot = copy.deepcopy(self.snapshot_manager.local_full_snapshot) original_snapshot = copy.deepcopy(mock_snapshot) mock_copy_path = self.test_file_1 + "_copy" @@ -1239,6 +1248,9 @@ def test_update_snapshot_copy(self): self.snapshot_manager.local_full_snapshot = original_snapshot def test_update_snapshot_move(self): + #inhibit _remove_snap_orphan function + self.snapshot_manager._remove_snap_orphan = self._void_function + original_snapshot = copy.deepcopy(self.snapshot_manager.local_full_snapshot) mock_snapshot = copy.deepcopy(original_snapshot) mock_new_dest = self.test_file_1 + "_new_dest" @@ -1254,6 +1266,9 @@ def test_update_snapshot_move(self): os.remove(mock_new_dest) def test_update_snapshot_delete(self): + #inhibit _remove_snap_orphan function + self.snapshot_manager._remove_snap_orphan = self._void_function + mock_snapshot = copy.deepcopy(self.snapshot_manager.local_full_snapshot) original_snapshot = copy.deepcopy(mock_snapshot) del mock_snapshot['fea80f2db003d4ebc4536023814aa885'] From 38763fef2ab66a68f54439163452a06f963eab1a Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Fri, 28 Nov 2014 15:59:40 +0100 Subject: [PATCH 4/7] remove redundant snapshot check --- client/client_daemon.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/client/client_daemon.py b/client/client_daemon.py index b3b1297..28168ac 100644 --- a/client/client_daemon.py +++ b/client/client_daemon.py @@ -868,18 +868,11 @@ def save_snapshot(self, timestamp): f.write( json.dumps({"timestamp": float(timestamp), "snapshot": self.last_status['snapshot']})) - def _remove_snap_orphan(self, md5_file): - self.local_full_snapshot[md5_file] = list(set(self.local_full_snapshot[md5_file])) - for path in list(self.local_full_snapshot[md5_file]): - if not os.path.exists(path): - self.local_full_snapshot[md5_file].remove(path) - def update_snapshot_upload(self, body): """ update of local full snapshot by upload request""" md5_file = self.file_snapMd5(body['src_path']) if md5_file in self.local_full_snapshot: self.local_full_snapshot[md5_file].append(get_relpath(body["src_path"])) - self._remove_snap_orphan(md5_file) else: self.local_full_snapshot[md5_file] = [get_relpath(body["src_path"])] @@ -892,7 +885,6 @@ def update_snapshot_update(self, body): # is a copy of another file self.local_full_snapshot[new_file_md5].append( get_relpath(body['src_path'])) - self._remove_snap_orphan(new_file_md5) else: # else create a new md5 self.local_full_snapshot[new_file_md5] = [get_relpath( @@ -903,7 +895,6 @@ def update_snapshot_copy(self, body): md5_file = self.file_snapMd5(body['src_path']) paths_of_file = self.local_full_snapshot[md5_file] paths_of_file.append(get_relpath(body["dst_path"])) - self._remove_snap_orphan(md5_file) def update_snapshot_move(self, body): """ update of local full snapshot by move request""" @@ -914,7 +905,6 @@ def update_snapshot_move(self, body): except ValueError: pass paths_of_file.append(get_relpath(body["dst_path"])) - self._remove_snap_orphan(md5_file) def update_snapshot_delete(self, body): """ update of local full snapshot by delete request""" From 099b033eaefbebf68eb1a05f4a46f004206b9a97 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Fri, 28 Nov 2014 16:40:32 +0100 Subject: [PATCH 5/7] add internal conflick error flag for sincronize dispatcher delete problem on daemon startup --- client/client_daemon.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/client/client_daemon.py b/client/client_daemon.py index 28168ac..801f721 100644 --- a/client/client_daemon.py +++ b/client/client_daemon.py @@ -805,6 +805,7 @@ def __init__(self, snapshot_file_path): self.snapshot_file_path = snapshot_file_path self.last_status = self._load_status() self.local_full_snapshot = self.instant_snapshot() + self.conflicted_sync = False def local_check(self): """ check id daemon is synchronized with local directory """ @@ -903,19 +904,21 @@ def update_snapshot_move(self, body): try: paths_of_file.remove(get_relpath(body["src_path"])) except ValueError: - pass + if not self.conflicted_sync: + raise paths_of_file.append(get_relpath(body["dst_path"])) def update_snapshot_delete(self, body): """ update of local full snapshot by delete request""" md5_file = self.find_file_md5(self.local_full_snapshot, get_relpath(body['src_path']), False) - paths_of_file = self.local_full_snapshot[md5_file] logger.debug("find md5: {}".format(md5_file)) if md5_file in self.local_full_snapshot: try: - paths_of_file.remove(get_relpath(body['src_path'])) + self.local_full_snapshot[md5_file].remove( + get_relpath(body['src_path'])) except ValueError: - pass + if not self.conflicted_sync: + raise if len(self.local_full_snapshot[md5_file]) == 0: del self.local_full_snapshot[md5_file] logger.debug("path deleted: " + get_relpath(body['src_path'])) @@ -970,6 +973,7 @@ def synchronize_dispatcher(self, server_timestamp, server_snapshot): command_list = [] #NO internal conflict if self.local_check(): # 1) + self.conflicted_sync = False if not self.is_syncro(server_timestamp): # 1) b. for new_server_path in new_server_paths: # 1) b 1 server_md5 = self.find_file_md5(server_snapshot, new_server_path) @@ -998,6 +1002,7 @@ def synchronize_dispatcher(self, server_timestamp, server_snapshot): #internal conflicts else: # 2) + self.conflicted_sync = True if self.is_syncro(server_timestamp): # 2) a logger.debug("****\tpush all\t****") for new_server_path in new_server_paths: # 2) a 1 @@ -1047,6 +1052,7 @@ def synchronize_dispatcher(self, server_timestamp, server_snapshot): for new_client_path in new_client_paths: # 2) b 3 logger.debug("remove remote\t{}".format(new_client_path)) command_list.append({'remote_delete': [new_client_path]}) + self.conflicted_sync = False return command_list From 84315ee950f40001cc1bbeca7a9853333098a8d8 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 1 Dec 2014 11:57:27 +0100 Subject: [PATCH 6/7] remove refuse in test --- client/test_client_daemon.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/client/test_client_daemon.py b/client/test_client_daemon.py index 131d812..1ccef42 100644 --- a/client/test_client_daemon.py +++ b/client/test_client_daemon.py @@ -1182,9 +1182,6 @@ def test_update_snapshot_upload(self): self.assertEqual(self.snapshot_manager.local_full_snapshot, original_snapshot) def test_update_snapshot_update(self): - #inhibit _remove_snap_orphan function - self.snapshot_manager._remove_snap_orphan = self._void_function - original_snapshot = copy.deepcopy(self.snapshot_manager.local_full_snapshot) mock_snapshot = copy.deepcopy(original_snapshot) mock_file_content = "test_content" @@ -1229,13 +1226,7 @@ def test_update_snapshot_update(self): } self.snapshotAsserEqual(self.snapshot_manager.local_full_snapshot, mock_snapshot) - def _void_function(self, *args, **kwargs): - pass - def test_update_snapshot_copy(self): - #inhibit _remove_snap_orphan function - self.snapshot_manager._remove_snap_orphan = self._void_function - mock_snapshot = copy.deepcopy(self.snapshot_manager.local_full_snapshot) original_snapshot = copy.deepcopy(mock_snapshot) mock_copy_path = self.test_file_1 + "_copy" @@ -1248,9 +1239,6 @@ def test_update_snapshot_copy(self): self.snapshot_manager.local_full_snapshot = original_snapshot def test_update_snapshot_move(self): - #inhibit _remove_snap_orphan function - self.snapshot_manager._remove_snap_orphan = self._void_function - original_snapshot = copy.deepcopy(self.snapshot_manager.local_full_snapshot) mock_snapshot = copy.deepcopy(original_snapshot) mock_new_dest = self.test_file_1 + "_new_dest" @@ -1266,9 +1254,6 @@ def test_update_snapshot_move(self): os.remove(mock_new_dest) def test_update_snapshot_delete(self): - #inhibit _remove_snap_orphan function - self.snapshot_manager._remove_snap_orphan = self._void_function - mock_snapshot = copy.deepcopy(self.snapshot_manager.local_full_snapshot) original_snapshot = copy.deepcopy(mock_snapshot) del mock_snapshot['fea80f2db003d4ebc4536023814aa885'] From 2cc8be185548a1d09c6311c36cda3547c8078ddf Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 1 Dec 2014 12:03:40 +0100 Subject: [PATCH 7/7] add comment --- client/client_daemon.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/client_daemon.py b/client/client_daemon.py index 801f721..196264a 100644 --- a/client/client_daemon.py +++ b/client/client_daemon.py @@ -805,6 +805,8 @@ def __init__(self, snapshot_file_path): self.snapshot_file_path = snapshot_file_path self.last_status = self._load_status() self.local_full_snapshot = self.instant_snapshot() + + #flag enabled only when there is a conflict in sinchronize. enable inhibit of some check self.conflicted_sync = False def local_check(self): @@ -904,6 +906,7 @@ def update_snapshot_move(self, body): try: paths_of_file.remove(get_relpath(body["src_path"])) except ValueError: + #if it is in internal conflict check sinchronize the error is inhibit if not self.conflicted_sync: raise paths_of_file.append(get_relpath(body["dst_path"])) @@ -917,6 +920,7 @@ def update_snapshot_delete(self, body): self.local_full_snapshot[md5_file].remove( get_relpath(body['src_path'])) except ValueError: + #if it is in internal conflict check sinchronize the error is inhibit if not self.conflicted_sync: raise if len(self.local_full_snapshot[md5_file]) == 0: