From 5d0fae1cbb927b1eaf9e4e4c9e4a531d0bc58576 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 1 Sep 2014 12:38:40 +0200 Subject: [PATCH 01/43] add system test: test_blackbox --- test_blackbox.py | 399 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 399 insertions(+) create mode 100644 test_blackbox.py diff --git a/test_blackbox.py b/test_blackbox.py new file mode 100644 index 0000000..e3deeb6 --- /dev/null +++ b/test_blackbox.py @@ -0,0 +1,399 @@ +#!/usr/bin/env python +#-*- coding: utf-8 -*- + +import subprocess +import time +import os +import shutil +import random +import ConfigParser +from passlib.hash import sha256_crypt +import json +import unittest +import filecmp +import hashlib +import string + +import select +import fcntl + +INIT_TIME = 5 +TRY_TIME = 5 +MAX_TRY = 3 + +PROCESS = [] + +TEST_ENV_DIR = os.path.join(os.path.expanduser('~'), 'TestEnv') + +# test user information: +# user: user1@test.it +# istance1: istance sinked with server +# istance2: istance of daemon not sinked with server + +CONF_USERS = { + 'istance1': { + 'username': 'user1@test.it', + 'password': 'password', + 'conf_path': os.path.join(TEST_ENV_DIR, 'istance1/user1@test.it_config'), + 'share_path': os.path.join(TEST_ENV_DIR, 'istance1/user1@test.it_share'), + 'dmn_command': 'cd {}; python {}/client/client_daemon.py'.format( + os.path.join(TEST_ENV_DIR, 'istance1/user1@test.it_config'), + os.path.dirname(os.path.abspath(__file__))) + }, + 'istance2': { + 'username': 'user1@test.it', + 'password': 'password', + 'conf_path': os.path.join(TEST_ENV_DIR, 'istance2/user1@test.it_config'), + 'share_path': os.path.join(TEST_ENV_DIR, 'istance2/user1@test.it_share'), + 'dmn_command': 'cd {}; python {}/client/client_daemon.py'.format( + os.path.join(TEST_ENV_DIR, 'istance2/user1@test.it_config'), + os.path.dirname(os.path.abspath(__file__))) + } +} + +# local server sincronized test directory for user "user1" +SERVER_USER_DATA = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "server", "user_data.json") +SERVER_USER_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'server/user_dirs/', + 'user1@test.it') + +SYNC_TIME = time.time() + + +def non_block_readline(output): + fd = output.fileno() + fl = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) + try: + return output.readline() + except IOError: + pass + + +def check_folder(src_folder, dst_folder): + for root, dirs, files in os.walk(src_folder): + for f in files: + full_src_path = os.path.join(root, f) + rel_path = os.path.relpath(full_src_path, src_folder) + full_dst_path = os.path.join(dst_folder, rel_path) + try: + result = filecmp.cmp(full_src_path, full_dst_path) + except OSError: + result = False + if not result: + print "---" + print "check propagation for: {}".format(rel_path) + print "from: {}".format(full_src_path) + print "to: {}".format(full_dst_path) + return result + return result + + +def check_propagation(first_folder, second_folder): + ''' + check identity betwen two specific folders contenute + ''' + # direct check + if not check_folder(first_folder, second_folder): + return False + + # reverse check + if not check_folder(second_folder, first_folder): + return False + + return True + + +def rand_content(size=4, chars=string.ascii_uppercase + string.digits): + ''' function for random string creation ''' + + return ''.join(random.choice(chars) for _ in range(size)) + + +def create_file(folder): + ''' + function for random file creation in a specific folder. + return the path of file and the md5 of content + ''' + while True: + filename = rand_content() + filepath = os.path.join(folder, filename) + if not os.path.exists(filepath): + content = rand_content(10) + open(filepath, 'w').write(content) + md5 = hashlib.md5(content).hexdigest() + return filepath, md5 + + +def create_folder_tree(main_path, user): + folders = ['mass_copy', 'mass_move', 'mass_delete', 'mass_modify'] + client_snap = {} + svr_filelist = {} + svr_filelist[''] = [ + user, + None, + SYNC_TIME + ] + + for folder in folders: + os.makedirs(os.path.join(main_path, folder)) + svr_filelist[folder] = [ + '/'.join([user, folder]), + None, + SYNC_TIME + ] + for i in range(10): + full_path, md5 = create_file(os.path.join(main_path, folder)) + rel_path = os.path.relpath(full_path, main_path) + if md5 in client_snap: + client_snap[md5].append(rel_path) + else: + client_snap[md5] = [rel_path] + + svr_filelist[rel_path] = [ + '/'.join([user, rel_path]), + md5, + SYNC_TIME + ] + + return client_snap, svr_filelist + + +def create_test_environment(): + if os.path.exists(TEST_ENV_DIR): + shutil.rmtree(TEST_ENV_DIR) + + if os.path.exists(SERVER_USER_DIR): + shutil.rmtree(SERVER_USER_DIR) + + os.makedirs(TEST_ENV_DIR) + os.makedirs(CONF_USERS['istance1']['conf_path']) + os.makedirs(CONF_USERS['istance1']['share_path']) + os.makedirs(CONF_USERS['istance2']['conf_path']) + os.makedirs(CONF_USERS['istance2']['share_path']) + + client_snap, svr_filelist = create_folder_tree( + CONF_USERS['istance1']['share_path'], + CONF_USERS['istance1']['username']) + shutil.copytree(CONF_USERS['istance1']['share_path'], SERVER_USER_DIR) + + # Create daemon config + port = 6666 + for k, v in CONF_USERS.items(): + daemon_ini = ConfigParser.ConfigParser() + daemon_ini.add_section('cmd') + daemon_ini.set('cmd', 'host', 'localhost') + daemon_ini.set('cmd', 'port', port) + daemon_ini.add_section('daemon_communication') + daemon_ini.set('daemon_communication', 'snapshot_file_path', 'snapshot_file.json') + daemon_ini.set('daemon_communication', 'dir_path', v['share_path']) + daemon_ini.set('daemon_communication', 'server_url', 'localhost') + daemon_ini.set('daemon_communication', 'server_port', 5000) + daemon_ini.set('daemon_communication', 'api_prefix', 'API/v1') + daemon_ini.set('daemon_communication', 'crash_repo_path', os.path.join(v['conf_path'], 'RawBox_crash_report.log')) + daemon_ini.set('daemon_communication', 'stdout_log_level', "DEBUG") + daemon_ini.set('daemon_communication', 'file_log_level', "ERROR") + daemon_ini.add_section('daemon_user_data') + daemon_ini.set('daemon_user_data', 'username', v['username']) + daemon_ini.set('daemon_user_data', 'password', v['password']) + daemon_ini.set('daemon_user_data', 'active', 'True') + + with open(os.path.join(v['conf_path'], "config.ini"), "w") as f: + daemon_ini.write(f) + port += 1 + + #snapshot for sinked instance + with open(os.path.join(CONF_USERS['istance1']['conf_path'], "snapshot_file.json"), "w") as f: + for k, v in client_snap.items(): + v.sort() + snap_list = sorted(list(client_snap.items())) + hashlib.md5(str(snap_list)).hexdigest() + snap = { + "timestamp": SYNC_TIME, + "snapshot": hashlib.md5(str(snap_list)).hexdigest(), + } + json.dump(snap, f) + + # Create server config + svr_conf = {} + svr_conf['users'] = { + CONF_USERS['istance1']['username']: { + "paths": svr_filelist, + "psw": sha256_crypt.encrypt(CONF_USERS['istance1']['password']), + "timestamp": SYNC_TIME, + }, + } + json.dump(svr_conf, open(SERVER_USER_DATA, 'w')) + + # Start subprocess + svr_process = subprocess.Popen( + "cd server; python server.py", + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + print "server PID: {}".format(svr_process.pid) + time.sleep(2) + + dmn_process = subprocess.Popen( + CONF_USERS['istance1']['dmn_command'], + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + print "daemon PID: {}".format(dmn_process.pid) + + time.sleep(INIT_TIME) + return svr_process, dmn_process + + +class BlackBoxTest(unittest.TestCase): + + @classmethod + def tearDownClass(cls): + for proc in PROCESS: + print '---' + print proc.pid + print proc.poll() + proc.kill() + print proc.poll() + + shutil.rmtree(TEST_ENV_DIR) + shutil.rmtree(SERVER_USER_DIR) + + def _check_folder(self, shared_path, server_path): + retry = 0 + while True: + print " try: {}".format(retry) + time.sleep(TRY_TIME) + try: + self.assertTrue( + check_propagation( + shared_path, + server_path)) + break + except AssertionError: + if retry >= MAX_TRY: + raise + else: + retry += 1 + + def test_mass_copy(self): + src_folder = os.path.join( + CONF_USERS['istance1']['share_path'], + 'mass_copy') + dst_folder = os.path.join( + CONF_USERS['istance1']['share_path'], + 'mass_copy_dst') + shutil.copytree( + src_folder, + dst_folder) + + self._check_folder( + CONF_USERS['istance1']['share_path'], + SERVER_USER_DIR) + + def test_mass_move(self): + src_folder = os.path.join( + CONF_USERS['istance1']['share_path'], + 'mass_move') + dst_folder = os.path.join( + CONF_USERS['istance1']['share_path'], + 'mass_move_dst') + os.makedirs(dst_folder) + shutil.move( + src_folder, + dst_folder) + + self._check_folder( + CONF_USERS['istance1']['share_path'], + SERVER_USER_DIR) + + def test_mass_delete(self): + del_folder = os.path.join( + CONF_USERS['istance1']['share_path'], + 'mass_delete') + shutil.rmtree(del_folder) + + self._check_folder( + CONF_USERS['istance1']['share_path'], + SERVER_USER_DIR) + + def test_mass_create(self): + crt_folder = os.path.join( + CONF_USERS['istance1']['share_path'], + 'mass_create') + os.makedirs(crt_folder) + for i in range(15): + create_file(crt_folder) + + self._check_folder( + CONF_USERS['istance1']['share_path'], + SERVER_USER_DIR) + + def test_mass_modify(self): + mod_folder = os.path.join( + CONF_USERS['istance1']['share_path'], + 'mass_modify') + for root, dirs, files in os.walk(mod_folder): + for f in files: + print f + filepath = os.path.join(root, f) + open(filepath, 'w').write(rand_content(15)) + + self._check_folder( + CONF_USERS['istance1']['share_path'], + SERVER_USER_DIR) + + def test_new_client(self): + dmn_process = subprocess.Popen( + CONF_USERS['istance2']['dmn_command'], + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + print "daemon PID: {}".format(dmn_process.pid) + PROCESS.append(dmn_process) + time.sleep(INIT_TIME) + + self._check_folder( + CONF_USERS['istance2']['share_path'], + SERVER_USER_DIR) + + +if __name__ == "__main__": + + svr_process, dmn_process = create_test_environment() + PROCESS.append(svr_process) + PROCESS.append(dmn_process) + + test = True + + if test: + unittest.main() + else: + out_list = [] + read_comm = {} + + for proc in PROCESS: + out_list.append(proc.stdout.fileno()) + read_comm[proc.stdout.fileno()] = proc.stdout + out_list.append(proc.stderr.fileno()) + read_comm[proc.stderr.fileno()] = proc.stderr + + for k in read_comm: + print 'fileno: {}'.format(k) + + while True: + try: + r, w, e = select.select(out_list, [], [], 5) + except select.error, err: + raise + + for fd in r: + message = non_block_readline(read_comm.get(fd, None)).strip() + if message != "": + print "read message: {}".format(fd) + print message + print "---" From 4b6ec10295e1020cf1a0da94894c76b1c80554c3 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 1 Sep 2014 12:40:42 +0200 Subject: [PATCH 02/43] temporary client_daemon refuse fix --- client/client_daemon.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/client/client_daemon.py b/client/client_daemon.py index 98dd5c5..30feeca 100644 --- a/client/client_daemon.py +++ b/client/client_daemon.py @@ -127,7 +127,7 @@ def download_file(self, dst_path): local_path = get_abspath(dst_path) if r.status_code == 200: - return local_path, r.text + return local_path, r.json() else: return False, False @@ -482,7 +482,6 @@ def load_config(): config_ini.set('daemon_communication', 'stdout_log_level', "DEBUG") config_ini.set('daemon_communication', 'file_log_level', "ERROR") - snapshot_file = config_ini.get('daemon_communication', 'snapshot_file_path') config = { "host": config_ini.get('cmd', 'host'), "port": config_ini.get('cmd', 'port'), @@ -495,15 +494,18 @@ def load_config(): "stdout_log_level": config_ini.get('daemon_communication', 'stdout_log_level'), "file_log_level": config_ini.get('daemon_communication', 'file_log_level'), "dir_path": config_ini.get('daemon_communication', 'dir_path'), - "snapshot_file_path": snapshot_file + "snapshot_file_path": config_ini.get('daemon_communication', 'snapshot_file_path') } try: os.makedirs(dir_path) except OSError: pass - with open(snapshot_file, 'w') as snapshot: + + if not os.path.exists(config["snapshot_file_path"]): + with open(config["snapshot_file_path"], 'w') as snapshot: json.dump({"timestamp": 0, "snapshot": ""}, snapshot) + try: config["username"] = config_ini.get('daemon_user_data', 'username') config["password"] = config_ini.get('daemon_user_data', 'password') @@ -966,7 +968,7 @@ def main(): while not user_exists: asyncore.poll(timeout=1.0) config, user_exists = load_config() - print config + server_com = ServerCommunicator( server_url=config['server_url'], username=config['username'], From cecf8d564ed4ea97a1d238fefa302d3880d5527c Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Wed, 3 Sep 2014 11:49:06 +0200 Subject: [PATCH 03/43] add file for flexible environment for system test (in progress) --- systemtest/testenvironment.py | 56 +++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 systemtest/testenvironment.py diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py new file mode 100644 index 0000000..6ffa7e7 --- /dev/null +++ b/systemtest/testenvironment.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +#-*- coding: utf-8 -*- + +import os + + +class EnvironmentManager(object): + + def __init__( + self, + dmn_src_path=os.path.join(os.path.dirname(os.path.abspath(__file__)), '/client'), + svr_src_path=os.path.join(os.path.dirname(os.path.abspath(__file__)), '/server/'), + dmn_test_dir=os.path.join(os.path.expanduser('~'), 'TestEnv'), + svr_usr_datafile=os.path.join(os.path.dirname(os.path.abspath(__file__)), "server", "user_data.json"), + svr_usr_dir=os.path.join(os.path.dirname(os.path.abspath(__file__)), 'server', 'user_dirs/') + ): + ''' + Create a folder tree for the execution of system test. + patameters: + dmn_src_path - daemnon and cmd_manager source path + default - os.path.join(os.path.dirname(os.path.abspath(__file__)),'/client/') + svr_src_path - server source path + default - os.path.join(os.path.dirname(os.path.abspath(__file__)),'/server/') + dmn_test_dir - test folder for inizialization daemon istance ambient. + for each daemon istance will create a dedicated folder that contain a config directory and a share directory + default - os.path.join(os.path.expanduser('~'), 'TestEnv') + svr_usr_datafile - path of server user svr_usr_datafile + default - os.path.join(os.path.dirname(os.path.abspath(__file__)), "server", "user_data.json") + svr_usr_dir - server path will contain user's synchronized folder + default - os.path.join(os.path.dirname(os.path.abspath(__file__), 'server/user_dirs/') + + Daemon's folder structure: + |root + -dmn_test_dir- + -istance_id- + -config- + -config.ini + -snapshot.json + -RawBox_crash_report.log + -share- + -file_to_share + Server's folder structure: + |root + -svr_usr_dir- + -username- + -file_to_shar + ''' + self.dmn_src_path = dmn_src_path + self.svr_src_path = svr_src_path + self.dmn_test_dir = dmn_test_dir + self.svr_usr_datafile = svr_usr_datafile + self.svr_usr_dir = svr_usr_dir + self.dmn_istance_list = [] + self.started_process = [] + + self.init_time = 5 From 4a1cb8fe2c1f6edf5cba98553459295f284007b2 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Tue, 9 Sep 2014 14:24:38 +0200 Subject: [PATCH 04/43] add init base structure creation --- systemtest/testenvironment.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 6ffa7e7..8efd455 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -2,6 +2,9 @@ #-*- coding: utf-8 -*- import os +import json +import time +import shutil class EnvironmentManager(object): @@ -50,7 +53,28 @@ def __init__( self.dmn_test_dir = dmn_test_dir self.svr_usr_datafile = svr_usr_datafile self.svr_usr_dir = svr_usr_dir - self.dmn_istance_list = [] + self.dmn_istance_list = {} + self.inc_id = 1 self.started_process = [] + self.sync_time = time.time() + + self.dmn_port = 6666 self.init_time = 5 + + #reset base environment tree + if os.path.exists(self.dmn_test_dir): + shutil.rmtree(self.dmn_test_dir) + + if os.path.exists(self.svr_usr_dir): + shutil.rmtree(self.svr_usr_dir) + + os.makedirs(self.dmn_test_dir) + os.makedirs(self.svr_usr_dir) + + with open(self.svr_usr_datafile, 'w') as datafile: + to_save = { + "users": {} + } + json.dump(to_save, datafile) + From d1db12110ad2721cd7c5d74747b2466fd815a63a Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Tue, 9 Sep 2014 15:18:06 +0200 Subject: [PATCH 05/43] add method for add user daemon istance --- systemtest/testenvironment.py | 66 +++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 8efd455..8e42dcc 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -5,6 +5,33 @@ import json import time import shutil +#import check password server function +from ...server.server import PasswordChecker as password_checker + +def check_password(password): + ''' + password checker adapter + parameters: + password - string + + return a Boolean + ''' + if password_checker(password) == password: + return True + else: + return False + + +def check_username(username): + ''' + username checker adapter + parameters: + username - string + + retunr a Boolean + ''' + email_regex = re.compile('[^@]+@[^@]+\.[^@]+') + return email_regex.match(username) class EnvironmentManager(object): @@ -78,3 +105,42 @@ def __init__( } json.dump(to_save, datafile) + def add_dmn_istance(self, ist_id=None, credential=None, svr_rec=False, dmn_rec=False): + ''' + add an istance at the daemon istance list + parameters: + id - identifier of istance + default - None, the id will be automatically assigned + credential - dict contain compatible username ("usr" key) and compatible password ("psw" key) of istance + exemple {"usr": "user@test.com", "psw": "Password<3"} + if svr_rec and dmn_rec are both False this var will be ignore + default - None, create a void istance + svr_rec - Boolean, mean the istance is registered on server yet. + default - False + dmn_rec - Boolean, mean the istance is logged in yet + can be True only if svr_rec are True + default - False + ''' + if not ist_id: + condition = True + while condition: + ist_id = ''.join(['ist_', self.inc_id]) + if ist_id not in self.dmn_istance_list: + condition = False + self.inc_id += 1 + else: + if ist_id in self.dmn_istance_list: + raise + + self.dmn_istance_list[ist_id] = { + 'svr_rec': svr_rec, + 'dmn_rec': dmn_rec, + } + + if svr_rec or dmn_rec: + if check_password(credential['psw']) and check_username(credential['usr']): + self.dmn_istance_list[ist_id]['usr'] = credential['usr'] + self.dmn_istance_list[ist_id]['psw'] = credential['psw'] + else: + self.dmn_istance_list[ist_id]['usr'] = None + self.dmn_istance_list[ist_id]['psw'] = None From 992c352200db234b772f39cfde08023425174c6b Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Tue, 9 Sep 2014 15:37:46 +0200 Subject: [PATCH 06/43] add methon for daemoin istance structure and configure propagation --- systemtest/testenvironment.py | 56 +++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 8e42dcc..050a716 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -4,7 +4,9 @@ import os import json import time +import ConfigParser import shutil +from passlib.hash import sha256_crypt #import check password server function from ...server.server import PasswordChecker as password_checker @@ -105,6 +107,60 @@ def __init__( } json.dump(to_save, datafile) + def _ist_propagation(self, ist_id): + # istance's folder tree creation + istance_path = os.path.join(self.dmn_test_dir, ist_id) + conf_path = os.path.join(istance_path, 'config') + share_path = os.path.join(istance_path, 'share') + os.makedirs(istance_path) + os.makedirs(conf_path) + os.makedirs(share_path) + + # istance's config file creation + daemon_ini = ConfigParser.ConfigParser() + daemon_ini.add_section('cmd') + daemon_ini.set('cmd', 'host', 'localhost') + daemon_ini.set('cmd', 'port', self.dmn_port) + daemon_ini.add_section('daemon_communication') + daemon_ini.set('daemon_communication', 'snapshot_file_path', 'snapshot_file.json') + daemon_ini.set('daemon_communication', 'dir_path', share_path) + daemon_ini.set('daemon_communication', 'server_url', 'localhost') + daemon_ini.set('daemon_communication', 'server_port', 5000) + daemon_ini.set('daemon_communication', 'api_prefix', 'API/v1') + daemon_ini.set('daemon_communication', 'crash_repo_path', os.path.join(conf_path, 'RawBox_crash_report.log')) + daemon_ini.set('daemon_communication', 'stdout_log_level', "DEBUG") + daemon_ini.set('daemon_communication', 'file_log_level', "ERROR") + daemon_ini.add_section('daemon_user_data') + daemon_ini.set('daemon_user_data', 'username', self.dmn_istance_list[ist_id]['usr']) + daemon_ini.set('daemon_user_data', 'password', self.dmn_istance_list[ist_id]['psw']) + daemon_ini.set('daemon_user_data', 'active', self.dmn_istance_list[ist_id]['dmn_rec']) + + with open(os.path.join(conf_path, 'config.ini'), 'w') as f: + daemon_ini.write(f) + self.dmn_port += 1 + + # istance's void snapshot file creation + open(os.path.join(conf_path, 'snapshot_file.json'), 'w').write('{}') + + # propagation in server user_data.json + if self.dmn_istance_list[ist_id]['svr_rec']: + with open(self.svr_usr_datafile, 'rw') as datafile: + svr_conf = json.load(datafile) + svr_conf['users'] = { + self.dmn_istance_list[ist_id]['usr']: { + "paths": { + '': [ + self.dmn_istance_list[ist_id]['usr'], + None, + self.sync_time + ] + }, + "psw": sha256_crypt.encrypt(self.dmn_istance_list[ist_id]['psw']), + "timestamp": self.sync_time, + }, + } + json.dump(svr_conf, datafile) + def add_dmn_istance(self, ist_id=None, credential=None, svr_rec=False, dmn_rec=False): ''' add an istance at the daemon istance list From 974100dcb7783acb1dfaed54b649f249f70d292c Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Tue, 9 Sep 2014 15:38:53 +0200 Subject: [PATCH 07/43] import for usarname checker error --- systemtest/testenvironment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 050a716..439ccbe 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -4,6 +4,7 @@ import os import json import time +import re import ConfigParser import shutil from passlib.hash import sha256_crypt From fc353fb7400354d5b197b2513a054d53881997fe Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Tue, 9 Sep 2014 15:40:37 +0200 Subject: [PATCH 08/43] add manage process function --- systemtest/testenvironment.py | 51 +++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 439ccbe..f648e16 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -10,6 +10,11 @@ from passlib.hash import sha256_crypt #import check password server function from ...server.server import PasswordChecker as password_checker +import signal +import subprocess + +INIT_TIME = 3 + def check_password(password): ''' @@ -37,6 +42,21 @@ def check_username(username): return email_regex.match(username) +def start_proc(command): + return subprocess.Popen( + command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + preexec_fn=os.setsid + ) + + +def terminate_proc(proc): + os.killpg(os.getpgid(proc), signal.SIGTERM) + + class EnvironmentManager(object): def __init__( @@ -108,6 +128,37 @@ def __init__( } json.dump(to_save, datafile) + def _start_serverproc(self): + ''' + start a server subprocess. The proc will be accessible in self.svr_process + ''' + command = 'python {}'.format(os.path.join(self.svr_src_path, 'server.py')) + self.svr_process = start_proc(command) + time.sleep(INIT_TIME) + + def _stop_serverproc(self): + ''' + stop a running server subprocess + ''' + if self.svr_process: + terminate_proc(self.svr_process) + + def _start_daemonproc(self, ist_id): + ''' + execute inizialization routine for the istance identified by ist_id: + it will propagate folder structure, config for daemon and server and it will start a daemon subprocess + ''' + if ist_id in self.dmn_istance_list: + command = 'cd {}; python {}'.format( + os.path.join(self.dmn_test_dir, ist_id, 'config'), + os.path.join(self.dmn_src_path, 'client', 'client_daermon.py') + ) + self.dmn_istance_list[ist_id]['process'] = start_proc(command) + + def _stop_daemonproc(self, ist_id): + if ist_id in self.dmn_istance_list and not self.dmn_istance_list[ist_id]['process'].poll(): + terminate_proc(self.dmn_istance_list[ist_id]['process']) + def _ist_propagation(self, ist_id): # istance's folder tree creation istance_path = os.path.join(self.dmn_test_dir, ist_id) From 7f1d81d6b0193bacfc941059620ea0cc35462b08 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Tue, 9 Sep 2014 15:41:02 +0200 Subject: [PATCH 09/43] minor docstring refactor --- systemtest/testenvironment.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index f648e16..5340e23 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -58,7 +58,7 @@ def terminate_proc(proc): class EnvironmentManager(object): - + def __init__( self, dmn_src_path=os.path.join(os.path.dirname(os.path.abspath(__file__)), '/client'), @@ -70,16 +70,16 @@ def __init__( ''' Create a folder tree for the execution of system test. patameters: - dmn_src_path - daemnon and cmd_manager source path + dmn_src_path - daemnon and cmd_manager source absolute path default - os.path.join(os.path.dirname(os.path.abspath(__file__)),'/client/') - svr_src_path - server source path + svr_src_path - server source absolute path default - os.path.join(os.path.dirname(os.path.abspath(__file__)),'/server/') - dmn_test_dir - test folder for inizialization daemon istance ambient. + dmn_test_dir - test absolute path of folder for inizialization daemon istance ambient. for each daemon istance will create a dedicated folder that contain a config directory and a share directory default - os.path.join(os.path.expanduser('~'), 'TestEnv') - svr_usr_datafile - path of server user svr_usr_datafile + svr_usr_datafile - server user svr_usr_datafile absolute path default - os.path.join(os.path.dirname(os.path.abspath(__file__)), "server", "user_data.json") - svr_usr_dir - server path will contain user's synchronized folder + svr_usr_dir - server abspath path will contain user's synchronized folder default - os.path.join(os.path.dirname(os.path.abspath(__file__), 'server/user_dirs/') Daemon's folder structure: From 749304a14abc1d57752799e35838fee8a44b85cc Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 15 Sep 2014 11:44:48 +0200 Subject: [PATCH 10/43] DirSnapshotManager take share dir by argument --- client/client_daemon.py | 6 ++++-- client/test_client_daemon.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/client/client_daemon.py b/client/client_daemon.py index 9c0a243..a1be801 100644 --- a/client/client_daemon.py +++ b/client/client_daemon.py @@ -648,9 +648,10 @@ def on_modified(self, event): class DirSnapshotManager(object): - def __init__(self, snapshot_file_path): + def __init__(self, snapshot_file_path, share_path): """ load the last global snapshot and create a instant_snapshot of local directory""" self.snapshot_file_path = snapshot_file_path + self.share_path = share_path self.last_status = self._load_status() self.local_full_snapshot = self.instant_snapshot() @@ -697,7 +698,7 @@ def instant_snapshot(self): """ create a snapshot of directory """ dir_snapshot = {} - for root, dirs, files in os.walk(CONFIG_DIR_PATH): + for root, dirs, files in os.walk(self.share_path): for f in files: full_path = os.path.join(root, f) file_md5 = self.file_snapMd5(full_path) @@ -980,6 +981,7 @@ def main(): snapshot_manager = DirSnapshotManager( snapshot_file_path=config['snapshot_file_path'], + share_path=config['dir_path'] ) server_com = ServerCommunicator( diff --git a/client/test_client_daemon.py b/client/test_client_daemon.py index 3992084..8f19242 100644 --- a/client/test_client_daemon.py +++ b/client/test_client_daemon.py @@ -849,7 +849,9 @@ def setUp(self): self.unsinked_timestamp = 123125 self.old_timestamp = 123122 - self.snapshot_manager = DirSnapshotManager(self.conf_snap_path) + self.snapshot_manager = DirSnapshotManager( + self.conf_snap_path, + self.test_share_dir) def tearDown(self): self.environment.remove() From 1f36923552be4c84fa54ce716ab5edb463ca6ec6 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 15 Sep 2014 11:45:15 +0200 Subject: [PATCH 11/43] fix test for json message reading --- client/test_client_daemon.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/client/test_client_daemon.py b/client/test_client_daemon.py index 8f19242..77cba67 100644 --- a/client/test_client_daemon.py +++ b/client/test_client_daemon.py @@ -388,7 +388,7 @@ def test_download(self): #check if methods are equal self.assertEqual(method, 'GET') #check response's body - self.assertEqual(response[1], u'[{"title": "Test"}]') + self.assertEqual(response[1], [{"title": "Test"}]) #Case: server bad request def _try_request(self, *args, **kwargs): @@ -609,9 +609,10 @@ def update_snapshot_upload(self, body): if not os.path.exists(self.client_path): os.makedirs(self.client_path) httpretty.enable() + test_json = json.dumps('stringa_test') httpretty.register_uri(httpretty.GET, 'http://localhost/api/v1/files/{}'.format(self.filename), - body='this is a test', - content_type='text/plain') + body='"this is a test"', + content_type='json') self.snapshot_manager = DirSnapshotManager() self.server_com = ServerCommunicator( server_url='http://localhost/api/v1', From ed676246eea492570d8962c146f1437699451528 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 15 Sep 2014 11:46:45 +0200 Subject: [PATCH 12/43] fix import and server check password --- systemtest/testenvironment.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 5340e23..e284ad1 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -8,8 +8,10 @@ import ConfigParser import shutil from passlib.hash import sha256_crypt + #import check password server function -from ...server.server import PasswordChecker as password_checker +from server import PasswordChecker + import signal import subprocess @@ -24,7 +26,7 @@ def check_password(password): return a Boolean ''' - if password_checker(password) == password: + if PasswordChecker(password) == password: return True else: return False From a19bb0f674167b2b1ac5e1d251f5c42b50bccf47 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 15 Sep 2014 11:47:26 +0200 Subject: [PATCH 13/43] fix default folder argument --- systemtest/testenvironment.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index e284ad1..e6cb950 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -63,26 +63,26 @@ class EnvironmentManager(object): def __init__( self, - dmn_src_path=os.path.join(os.path.dirname(os.path.abspath(__file__)), '/client'), - svr_src_path=os.path.join(os.path.dirname(os.path.abspath(__file__)), '/server/'), + dmn_src_path=os.path.join(os.getcwd(), 'client'), + svr_src_path=os.path.join(os.getcwd(), 'server'), dmn_test_dir=os.path.join(os.path.expanduser('~'), 'TestEnv'), - svr_usr_datafile=os.path.join(os.path.dirname(os.path.abspath(__file__)), "server", "user_data.json"), - svr_usr_dir=os.path.join(os.path.dirname(os.path.abspath(__file__)), 'server', 'user_dirs/') + svr_usr_datafile=os.path.join(os.getcwd(), "server", "user_data.json"), + svr_usr_dir=os.path.join(os.getcwd(), 'server', 'user_dirs') ): ''' Create a folder tree for the execution of system test. patameters: dmn_src_path - daemnon and cmd_manager source absolute path - default - os.path.join(os.path.dirname(os.path.abspath(__file__)),'/client/') + default - os.path.join(os.getcwd(),'/client/') svr_src_path - server source absolute path - default - os.path.join(os.path.dirname(os.path.abspath(__file__)),'/server/') + default - os.path.join(os.getcwd(),'/server/') dmn_test_dir - test absolute path of folder for inizialization daemon istance ambient. for each daemon istance will create a dedicated folder that contain a config directory and a share directory default - os.path.join(os.path.expanduser('~'), 'TestEnv') svr_usr_datafile - server user svr_usr_datafile absolute path - default - os.path.join(os.path.dirname(os.path.abspath(__file__)), "server", "user_data.json") + default - os.path.join(os.getcwd(), "server", "user_data.json") svr_usr_dir - server abspath path will contain user's synchronized folder - default - os.path.join(os.path.dirname(os.path.abspath(__file__), 'server/user_dirs/') + default - os.path.join(os.getcwd(), 'server/user_dirs/') Daemon's folder structure: |root From e586bcfe389b638b84c0ad95dba7491681cfa5dd Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 15 Sep 2014 12:01:42 +0200 Subject: [PATCH 14/43] add_daemon_istance create config folder, check and init istance dict value --- systemtest/testenvironment.py | 64 +++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index e6cb950..20bc984 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -57,6 +57,16 @@ def start_proc(command): def terminate_proc(proc): os.killpg(os.getpgid(proc), signal.SIGTERM) +class InputError(Exception): + ''' + Exception raised for errors in the input parameters + ''' + + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) class EnvironmentManager(object): @@ -162,13 +172,6 @@ def _stop_daemonproc(self, ist_id): terminate_proc(self.dmn_istance_list[ist_id]['process']) def _ist_propagation(self, ist_id): - # istance's folder tree creation - istance_path = os.path.join(self.dmn_test_dir, ist_id) - conf_path = os.path.join(istance_path, 'config') - share_path = os.path.join(istance_path, 'share') - os.makedirs(istance_path) - os.makedirs(conf_path) - os.makedirs(share_path) # istance's config file creation daemon_ini = ConfigParser.ConfigParser() @@ -228,29 +231,56 @@ def add_dmn_istance(self, ist_id=None, credential=None, svr_rec=False, dmn_rec=F svr_rec - Boolean, mean the istance is registered on server yet. default - False dmn_rec - Boolean, mean the istance is logged in yet - can be True only if svr_rec are True + can be True only if svr_rec is True default - False + return: string istance id ''' if not ist_id: condition = True while condition: - ist_id = ''.join(['ist_', self.inc_id]) + ist_id = 'ist_{}'.format(self.inc_id) if ist_id not in self.dmn_istance_list: condition = False self.inc_id += 1 else: if ist_id in self.dmn_istance_list: - raise + raise InputError('id not valid') - self.dmn_istance_list[ist_id] = { - 'svr_rec': svr_rec, - 'dmn_rec': dmn_rec, - } + self.dmn_istance_list[ist_id] = {} + self.dmn_istance_list[ist_id]['dmn_port'] = self.dmn_port + self.dmn_port += 1 - if svr_rec or dmn_rec: + if svr_rec: if check_password(credential['psw']) and check_username(credential['usr']): self.dmn_istance_list[ist_id]['usr'] = credential['usr'] self.dmn_istance_list[ist_id]['psw'] = credential['psw'] + self.dmn_istance_list[ist_id]['dmn_rec'] = dmn_rec else: - self.dmn_istance_list[ist_id]['usr'] = None - self.dmn_istance_list[ist_id]['psw'] = None + raise InputError('credential not valid') + else: + self.dmn_istance_list[ist_id]['usr'] = None + self.dmn_istance_list[ist_id]['psw'] = None + self.dmn_istance_list[ist_id]['dmn_rec'] = False + self.dmn_istance_list[ist_id]['svr_rec'] = svr_rec + + self.sync_dmn_share(ist_id) + + istance_path = os.path.join(self.dmn_test_dir, ist_id) + conf_path = os.path.join(istance_path, 'config') + share_path = os.path.join(istance_path, 'share') + self.dmn_istance_list[ist_id]['root_path'] = istance_path + self.dmn_istance_list[ist_id]['conf_path'] = conf_path + self.dmn_istance_list[ist_id]['share_path'] = share_path + + if os.path.exists(istance_path): + shutil.rmtree(istance_path) + if os.path.exists(conf_path): + shutil.rmtree(conf_path) + if os.path.exists(share_path): + shutil.rmtree(share_path) + os.makedirs(istance_path) + os.makedirs(conf_path) + os.makedirs(share_path) + + return ist_id + From a8be5b05f7d9b4da4349478a41d3512773448a99 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 15 Sep 2014 12:05:06 +0200 Subject: [PATCH 15/43] istance propagation method refactoring --- systemtest/testenvironment.py | 125 +++++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 40 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 20bc984..ed143df 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -12,6 +12,9 @@ #import check password server function from server import PasswordChecker +#import client daemon snapshot manager object +from client import DirSnapshotManager + import signal import subprocess @@ -57,6 +60,76 @@ def start_proc(command): def terminate_proc(proc): os.killpg(os.getpgid(proc), signal.SIGTERM) +def update_srv_userdata_adt(ist_information, svr_datastorage, sync_time, svr_usr_dir): + ''' + save the init user information on the server userdata storage system + parameters: + ist_information - dict with the daemon istance information + as EnvironmentManager's dmn_istance_list + svr_datastorage - server storage reference + sync_time - timestamp of synchronization + ''' + try: + svr_conf = json.load(open(svr_datastorage, 'r')) + except IOError: + svr_conf = { + "users": {} + } + + svr_conf['users'][ist_information['usr']] = { + "paths": { + '': [ + ist_information['usr'], + None, + sync_time + ] + }, + "psw": sha256_crypt.encrypt(ist_information['psw']), + "timestamp": sync_time, + } + if 'svr_filelist' in ist_information: + shutil.copytree( + ist_information['share_path'], + os.path.join(svr_usr_dir, ist_information['usr'])) + svr_conf['users'][ist_information['usr']]['paths'].update( + ist_information['svr_filelist']) + json.dump(svr_conf, open(svr_datastorage, 'w')) + + +def create_ist_conf_file(ist_information): + ''' + create a daemon config.ini file in the configuration path + ''' + daemon_ini = ConfigParser.ConfigParser() + daemon_ini.add_section('cmd') + daemon_ini.set('cmd', 'host', 'localhost') + daemon_ini.set('cmd', 'port', ist_information['dmn_port']) + daemon_ini.add_section('daemon_communication') + daemon_ini.set('daemon_communication', 'snapshot_file_path', 'snapshot_file.json') + daemon_ini.set('daemon_communication', 'dir_path', ist_information['share_path']) + daemon_ini.set('daemon_communication', 'server_url', 'localhost') + daemon_ini.set('daemon_communication', 'server_port', 5000) + daemon_ini.set('daemon_communication', 'api_prefix', 'API/v1') + daemon_ini.set('daemon_communication', 'crash_repo_path', os.path.join(ist_information['conf_path'], 'RawBox_crash_report.log')) + daemon_ini.set('daemon_communication', 'stdout_log_level', "DEBUG") + daemon_ini.set('daemon_communication', 'file_log_level', "ERROR") + daemon_ini.add_section('daemon_user_data') + daemon_ini.set('daemon_user_data', 'username', ist_information['usr']) + daemon_ini.set('daemon_user_data', 'password', ist_information['psw']) + daemon_ini.set('daemon_user_data', 'active', ist_information['dmn_rec']) + + with open(os.path.join(ist_information['conf_path'], 'config.ini'), 'w') as f: + daemon_ini.write(f) + + +def create_spanshot_file(share_path, timestamp, snap_path, synced=True): + open(snap_path, 'w').write('{}') + + if synced: + snap_manager = DirSnapshotManager(snap_path, share_path) + snap_manager.save_snapshot(timestamp) + + class InputError(Exception): ''' Exception raised for errors in the input parameters @@ -172,51 +245,23 @@ def _stop_daemonproc(self, ist_id): terminate_proc(self.dmn_istance_list[ist_id]['process']) def _ist_propagation(self, ist_id): - # istance's config file creation - daemon_ini = ConfigParser.ConfigParser() - daemon_ini.add_section('cmd') - daemon_ini.set('cmd', 'host', 'localhost') - daemon_ini.set('cmd', 'port', self.dmn_port) - daemon_ini.add_section('daemon_communication') - daemon_ini.set('daemon_communication', 'snapshot_file_path', 'snapshot_file.json') - daemon_ini.set('daemon_communication', 'dir_path', share_path) - daemon_ini.set('daemon_communication', 'server_url', 'localhost') - daemon_ini.set('daemon_communication', 'server_port', 5000) - daemon_ini.set('daemon_communication', 'api_prefix', 'API/v1') - daemon_ini.set('daemon_communication', 'crash_repo_path', os.path.join(conf_path, 'RawBox_crash_report.log')) - daemon_ini.set('daemon_communication', 'stdout_log_level', "DEBUG") - daemon_ini.set('daemon_communication', 'file_log_level', "ERROR") - daemon_ini.add_section('daemon_user_data') - daemon_ini.set('daemon_user_data', 'username', self.dmn_istance_list[ist_id]['usr']) - daemon_ini.set('daemon_user_data', 'password', self.dmn_istance_list[ist_id]['psw']) - daemon_ini.set('daemon_user_data', 'active', self.dmn_istance_list[ist_id]['dmn_rec']) - - with open(os.path.join(conf_path, 'config.ini'), 'w') as f: - daemon_ini.write(f) - self.dmn_port += 1 + create_ist_conf_file(self.dmn_istance_list[ist_id]) - # istance's void snapshot file creation - open(os.path.join(conf_path, 'snapshot_file.json'), 'w').write('{}') + # istance's stapshot file creation + create_spanshot_file( + self.dmn_istance_list[ist_id]['share_path'], + self.sync_time, + os.path.join(self.dmn_istance_list[ist_id]['conf_path'], 'snapshot_file.json'), + self.dmn_istance_list[ist_id]['self_sync']) # propagation in server user_data.json if self.dmn_istance_list[ist_id]['svr_rec']: - with open(self.svr_usr_datafile, 'rw') as datafile: - svr_conf = json.load(datafile) - svr_conf['users'] = { - self.dmn_istance_list[ist_id]['usr']: { - "paths": { - '': [ - self.dmn_istance_list[ist_id]['usr'], - None, - self.sync_time - ] - }, - "psw": sha256_crypt.encrypt(self.dmn_istance_list[ist_id]['psw']), - "timestamp": self.sync_time, - }, - } - json.dump(svr_conf, datafile) + update_srv_userdata_adt( + self.dmn_istance_list[ist_id], + self.svr_usr_datafile, + self.sync_time, + self.svr_usr_dir) def add_dmn_istance(self, ist_id=None, credential=None, svr_rec=False, dmn_rec=False): ''' From 77e2df771465e1aee834d84db259f613886432cd Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 15 Sep 2014 12:05:47 +0200 Subject: [PATCH 16/43] fix terminate proc pid error --- systemtest/testenvironment.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index ed143df..2de2573 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -59,7 +59,14 @@ def start_proc(command): def terminate_proc(proc): - os.killpg(os.getpgid(proc), signal.SIGTERM) + ''' + send SIGTERM to process + paramenters: + proc - subprocess object + ''' + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + + def update_srv_userdata_adt(ist_information, svr_datastorage, sync_time, svr_usr_dir): ''' save the init user information on the server userdata storage system From 49bad7b8824d3e7e3d9658c7feeb6106b732e6b4 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 15 Sep 2014 12:06:30 +0200 Subject: [PATCH 17/43] fix daemon command creation --- systemtest/testenvironment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 2de2573..91df7be 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -243,7 +243,7 @@ def _start_daemonproc(self, ist_id): if ist_id in self.dmn_istance_list: command = 'cd {}; python {}'.format( os.path.join(self.dmn_test_dir, ist_id, 'config'), - os.path.join(self.dmn_src_path, 'client', 'client_daermon.py') + os.path.join(self.dmn_src_path, 'client_daemon.py') ) self.dmn_istance_list[ist_id]['process'] = start_proc(command) From 2fe6b6b218265757be2c8db381a994d28ebd4700 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 15 Sep 2014 12:07:37 +0200 Subject: [PATCH 18/43] delete useless creation and init refuse --- systemtest/testenvironment.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 91df7be..bbeb72b 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -214,19 +214,12 @@ def __init__( os.makedirs(self.dmn_test_dir) os.makedirs(self.svr_usr_dir) - with open(self.svr_usr_datafile, 'w') as datafile: - to_save = { - "users": {} - } - json.dump(to_save, datafile) - def _start_serverproc(self): ''' start a server subprocess. The proc will be accessible in self.svr_process ''' command = 'python {}'.format(os.path.join(self.svr_src_path, 'server.py')) self.svr_process = start_proc(command) - time.sleep(INIT_TIME) def _stop_serverproc(self): ''' From 40ab9afe02bce556568456941d4258c4ef496a23 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 15 Sep 2014 12:08:58 +0200 Subject: [PATCH 19/43] sync share methon for set parameter before propagation --- systemtest/testenvironment.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index bbeb72b..a1546a1 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -329,3 +329,16 @@ def add_dmn_istance(self, ist_id=None, credential=None, svr_rec=False, dmn_rec=F return ist_id + def sync_dmn_share(self, ist_id, self_sync=True, server_sync=True): + ''' + synchronize istance share folder in the configuration file. + Attributes: + ist_id - istance id + self_sync - boolean. if true it will synchronize the snapshot file + default True + server_sync - boolean. if true it will synchronize the server user data + default True + ''' + self.dmn_istance_list[ist_id]['self_sync'] = self_sync + self.dmn_istance_list[ist_id]['server_sync'] = server_sync + From 71d89397d5dba0ac3521e3238d8a16215d67ad78 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 15 Sep 2014 12:10:30 +0200 Subject: [PATCH 20/43] add random file method to specific folder --- systemtest/testenvironment.py | 45 +++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index a1546a1..f3f9725 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -15,10 +15,32 @@ #import client daemon snapshot manager object from client import DirSnapshotManager +import string +import random +import hashlib import signal import subprocess INIT_TIME = 3 +def rand_content(size=4, chars=string.ascii_uppercase + string.digits): + ''' function for random string creation ''' + + return ''.join(random.choice(chars) for _ in range(size)) + + +def create_file(folder): + ''' + function for random file creation in a specific folder. + return the path of file and the md5 of content + ''' + while True: + filename = rand_content() + filepath = os.path.join(folder, filename) + if not os.path.exists(filepath): + content = rand_content(10) + open(filepath, 'w').write(content) + md5 = hashlib.md5(content).hexdigest() + return filepath, md5 def check_password(password): @@ -342,3 +364,26 @@ def sync_dmn_share(self, ist_id, self_sync=True, server_sync=True): self.dmn_istance_list[ist_id]['self_sync'] = self_sync self.dmn_istance_list[ist_id]['server_sync'] = server_sync + def add_rndfile_to_ist(self, ist_id, num_file=10, relpath=None): + ''' + add text random files to specified istance's relpath + ''' + if relpath: + main_path = os.path.join( + self.dmn_istance_list[ist_id]['share_path'], + relpath) + else: + main_path = self.dmn_istance_list[ist_id]['share_path'] + + if 'svr_filelist' not in self.dmn_istance_list[ist_id]: + self.dmn_istance_list[ist_id]['svr_filelist'] = {} + + for e_file in range(num_file): + full_path, md5 = create_file(main_path) + rel_path = os.path.relpath(full_path, main_path) + self.dmn_istance_list[ist_id]['svr_filelist'][rel_path] = [ + os.path.join(self.dmn_istance_list[ist_id]['usr'], rel_path), + md5, + self.sync_time + ] + From 2e35883aae9bc3cc9eb08f0a5b8760d378eaaff7 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Tue, 16 Sep 2014 10:44:36 +0200 Subject: [PATCH 21/43] add start all and stop all method --- systemtest/testenvironment.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index f3f9725..4589f21 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -21,7 +21,7 @@ import signal import subprocess -INIT_TIME = 3 + def rand_content(size=4, chars=string.ascii_uppercase + string.digits): ''' function for random string creation ''' @@ -219,12 +219,11 @@ def __init__( self.svr_usr_dir = svr_usr_dir self.dmn_istance_list = {} self.inc_id = 1 - self.started_process = [] self.sync_time = time.time() self.dmn_port = 6666 - self.init_time = 5 + self.init_time = 3 #reset base environment tree if os.path.exists(self.dmn_test_dir): @@ -285,6 +284,27 @@ def _ist_propagation(self, ist_id): self.sync_time, self.svr_usr_dir) + def start_test_environment(self): + # daemon settings propagation + for ist_id in self.dmn_istance_list: + self._ist_propagation(ist_id) + + # start server process + self._start_serverproc() + time.sleep(self.init_time) + + # start daemon process + for ist_id in self.dmn_istance_list: + self._start_daemonproc(ist_id) + + def stop_test_environment(self): + # stop daemon process + for ist_id in self.dmn_istance_list: + self._stop_daemonproc(ist_id) + + #stop server process + self._stop_serverproc() + def add_dmn_istance(self, ist_id=None, credential=None, svr_rec=False, dmn_rec=False): ''' add an istance at the daemon istance list From dc795be300c7c1fe070ddf55bac4085051e8a496 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Tue, 16 Sep 2014 10:45:05 +0200 Subject: [PATCH 22/43] add flush method for remove all environment test file --- systemtest/testenvironment.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 4589f21..252d2dd 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -305,6 +305,23 @@ def stop_test_environment(self): #stop server process self._stop_serverproc() + def flush(self): + ''' + remove environment file structure + ''' + try: + shutil.rmtree(self.dmn_test_dir) + except OSError: + pass + try: + shutil.rmtree(self.svr_usr_dir) + except OSError: + pass + try: + os.remove(self.svr_usr_datafile) + except OSError: + pass + def add_dmn_istance(self, ist_id=None, credential=None, svr_rec=False, dmn_rec=False): ''' add an istance at the daemon istance list From 2b1f578064183091c55c11cccd96bbb8b2ed14ba Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Tue, 16 Sep 2014 10:45:25 +0200 Subject: [PATCH 23/43] add docstring --- systemtest/testenvironment.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 252d2dd..e8f2416 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -70,6 +70,12 @@ def check_username(username): def start_proc(command): + ''' + start subprocess + parameters: + command - shell command to start + return subprocess object + ''' return subprocess.Popen( command, shell=True, From 70257cb4ce6f270bfc3166869dcfa1dc2090fa30 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Fri, 19 Sep 2014 15:57:06 +0200 Subject: [PATCH 24/43] fix relative path problem --- systemtest/testenvironment.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index e8f2416..57d89c4 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -412,21 +412,28 @@ def add_rndfile_to_ist(self, ist_id, num_file=10, relpath=None): add text random files to specified istance's relpath ''' if relpath: - main_path = os.path.join( + dst_path = os.path.join( self.dmn_istance_list[ist_id]['share_path'], relpath) else: - main_path = self.dmn_istance_list[ist_id]['share_path'] + dst_path = self.dmn_istance_list[ist_id]['share_path'] if 'svr_filelist' not in self.dmn_istance_list[ist_id]: self.dmn_istance_list[ist_id]['svr_filelist'] = {} for e_file in range(num_file): - full_path, md5 = create_file(main_path) - rel_path = os.path.relpath(full_path, main_path) - self.dmn_istance_list[ist_id]['svr_filelist'][rel_path] = [ - os.path.join(self.dmn_istance_list[ist_id]['usr'], rel_path), - md5, - self.sync_time - ] + filename, full_path, md5 = create_file(dst_path) + rel_path = os.path.relpath( + full_path, + self.dmn_istance_list[ist_id]['share_path']) + + if filename not in self.dmn_istance_list[ist_id]['svr_filelist']: + self.dmn_istance_list[ist_id]['svr_filelist'][rel_path] = [] + + self.dmn_istance_list[ist_id]['svr_filelist'][rel_path].append( + os.path.join(self.dmn_istance_list[ist_id]['usr'], rel_path)) + self.dmn_istance_list[ist_id]['svr_filelist'][rel_path].append(md5) + self.dmn_istance_list[ist_id]['svr_filelist'][rel_path].append( + self.sync_time) + From c8c9bbc1491f421e134c8a6d9bf4f9cf66e58e77 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Fri, 19 Sep 2014 15:58:00 +0200 Subject: [PATCH 25/43] add method for adding custom folder --- systemtest/testenvironment.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 57d89c4..48e71f6 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -407,6 +407,24 @@ def sync_dmn_share(self, ist_id, self_sync=True, server_sync=True): self.dmn_istance_list[ist_id]['self_sync'] = self_sync self.dmn_istance_list[ist_id]['server_sync'] = server_sync + def add_fld_to_ist(self, ist_id, folder): + ''' + add new folder in istance's share directory identified by ist_id + ''' + path = os.path.join( + self.dmn_istance_list[ist_id]['share_path'], + folder) + if not os.path.exists(path): + os.makedirs(path) + if 'svr_filelist' not in self.dmn_istance_list[ist_id]: + self.dmn_istance_list[ist_id]['svr_filelist'] = {} + self.dmn_istance_list[ist_id]['svr_filelist'][folder] = [ + os.path.join(self.dmn_istance_list[ist_id]['usr'], folder), + None, + self.sync_time + ] + return path + def add_rndfile_to_ist(self, ist_id, num_file=10, relpath=None): ''' add text random files to specified istance's relpath From 362baea45972105ff7a617c047346d753928071c Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Fri, 19 Sep 2014 16:00:08 +0200 Subject: [PATCH 26/43] create a dedicated unittest class for system test --- systemtest/testenvironment.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 48e71f6..77d0eaf 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -21,6 +21,7 @@ import signal import subprocess +import unittest def rand_content(size=4, chars=string.ascii_uppercase + string.digits): ''' function for random string creation ''' @@ -455,3 +456,20 @@ def add_rndfile_to_ist(self, ist_id, num_file=10, relpath=None): self.sync_time) +class BlackBoxTest(unittest.TestCase): + ''' + expand unittest functionality: + add inizialization of EnvironmentManager on setUp method + self.env + stop all subprocess and flush all temporary test file on tearDown method + + add _check_folder method for testing share folder event propagation + ''' + + def setUp(self): + self.env = EnvironmentManager() + + def tearDown(self): + self.env.stop_test_environment() + self.env.flush() + From 7b215364d3a4e070b7b62ce68ec15fb42524cf57 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Fri, 19 Sep 2014 16:01:24 +0200 Subject: [PATCH 27/43] fix for refuse in utility function --- systemtest/testenvironment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 77d0eaf..5e39ddd 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -41,7 +41,7 @@ def create_file(folder): content = rand_content(10) open(filepath, 'w').write(content) md5 = hashlib.md5(content).hexdigest() - return filepath, md5 + return filename, filepath, md5 def check_password(password): From 727288c12fba02f89c6de14e29ac670eaa83ff8c Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Fri, 19 Sep 2014 16:04:23 +0200 Subject: [PATCH 28/43] add a check propagation system in unittest method --- systemtest/testenvironment.py | 57 +++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 5e39ddd..8e7b6e9 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -22,6 +22,8 @@ import subprocess import unittest +import filecmp + def rand_content(size=4, chars=string.ascii_uppercase + string.digits): ''' function for random string creation ''' @@ -456,6 +458,40 @@ def add_rndfile_to_ist(self, ist_id, num_file=10, relpath=None): self.sync_time) +def check_folder(src_folder, dst_folder): + for root, dirs, files in os.walk(src_folder): + for f in files: + full_src_path = os.path.join(root, f) + rel_path = os.path.relpath(full_src_path, src_folder) + full_dst_path = os.path.join(dst_folder, rel_path) + try: + result = filecmp.cmp(full_src_path, full_dst_path) + except OSError: + result = False + if not result: + print "---" + print "check propagation for: {}".format(rel_path) + print "from: {}".format(full_src_path) + print "to: {}".format(full_dst_path) + return result + return result + + +def check_propagation(first_folder, second_folder): + ''' + check identity betwen two specific folders contenute + ''' + # direct check + if not check_folder(first_folder, second_folder): + return False + + # reverse check + if not check_folder(second_folder, first_folder): + return False + + return True + + class BlackBoxTest(unittest.TestCase): ''' expand unittest functionality: @@ -468,8 +504,29 @@ class BlackBoxTest(unittest.TestCase): def setUp(self): self.env = EnvironmentManager() + self.num_try = 3 + self.wait_time = 1 def tearDown(self): self.env.stop_test_environment() self.env.flush() + def _check_folder(self): + cron = time.time() + for key, ist in self.env.dmn_istance_list.items(): + retry = 0 + while True: + print " try: {}".format(retry) + time.sleep(self.wait_time) + try: + self.assertTrue( + check_propagation( + ist['share_path'], + os.path.join(self.env.svr_usr_dir, ist['usr']))) + break + except AssertionError: + if retry >= self.num_try: + raise + else: + retry += 1 + print "check time: {}".format(time.time() - cron) From 62bfca58f1e53b0fb11ec5bbb6371994f1d4edf7 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Fri, 19 Sep 2014 16:11:45 +0200 Subject: [PATCH 29/43] add get_share_path utility --- systemtest/testenvironment.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 8e7b6e9..320c1ca 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -314,6 +314,9 @@ def stop_test_environment(self): #stop server process self._stop_serverproc() + def get_share_path(self, ist_id): + return self.dmn_istance_list[ist_id]['share_path'] + def flush(self): ''' remove environment file structure From e57406d27d8301dd657d63368a69cb1b8e26169f Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Tue, 23 Sep 2014 11:55:45 +0200 Subject: [PATCH 30/43] minor test enviromnent fix --- systemtest/testenvironment.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 320c1ca..d6e900d 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -131,6 +131,8 @@ def update_srv_userdata_adt(ist_information, svr_datastorage, sync_time, svr_usr os.path.join(svr_usr_dir, ist_information['usr'])) svr_conf['users'][ist_information['usr']]['paths'].update( ist_information['svr_filelist']) + else: + os.makedirs(os.path.join(svr_usr_dir, ist_information['usr'])) json.dump(svr_conf, open(svr_datastorage, 'w')) @@ -305,6 +307,7 @@ def start_test_environment(self): # start daemon process for ist_id in self.dmn_istance_list: self._start_daemonproc(ist_id) + time.sleep(self.init_time) def stop_test_environment(self): # stop daemon process @@ -462,6 +465,7 @@ def add_rndfile_to_ist(self, ist_id, num_file=10, relpath=None): def check_folder(src_folder, dst_folder): + result = True for root, dirs, files in os.walk(src_folder): for f in files: full_src_path = os.path.join(root, f) @@ -516,7 +520,7 @@ def tearDown(self): def _check_folder(self): cron = time.time() - for key, ist in self.env.dmn_istance_list.items(): + for key, ist in self.env.dmn_istance_list.iteritems(): retry = 0 while True: print " try: {}".format(retry) From 38beb6c2099b6b5454afae51f8f32b3c08715221 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Tue, 23 Sep 2014 11:56:41 +0200 Subject: [PATCH 31/43] docstring and user interface print refactor --- systemtest/testenvironment.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index d6e900d..1fbebff 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -296,6 +296,9 @@ def _ist_propagation(self, ist_id): self.svr_usr_dir) def start_test_environment(self): + '''Start all enviromnent server and istance process + ''' + print '\n--- START test enviromnent process' # daemon settings propagation for ist_id in self.dmn_istance_list: self._ist_propagation(ist_id) @@ -308,14 +311,19 @@ def start_test_environment(self): for ist_id in self.dmn_istance_list: self._start_daemonproc(ist_id) time.sleep(self.init_time) + print 'Started' def stop_test_environment(self): + '''Stop all enviromnent server and istance process + ''' + print 'STOP test environment process' # stop daemon process for ist_id in self.dmn_istance_list: self._stop_daemonproc(ist_id) #stop server process self._stop_serverproc() + print 'Stopped' def get_share_path(self, ist_id): return self.dmn_istance_list[ist_id]['share_path'] @@ -476,8 +484,7 @@ def check_folder(src_folder, dst_folder): except OSError: result = False if not result: - print "---" - print "check propagation for: {}".format(rel_path) + print "ERROR check propagation for: {}".format(rel_path) print "from: {}".format(full_src_path) print "to: {}".format(full_dst_path) return result @@ -519,6 +526,7 @@ def tearDown(self): self.env.flush() def _check_folder(self): + print '\n---- start checking folder' cron = time.time() for key, ist in self.env.dmn_istance_list.iteritems(): retry = 0 @@ -536,4 +544,4 @@ def _check_folder(self): raise else: retry += 1 - print "check time: {}".format(time.time() - cron) + print '\nSuccess\n check time: {}\n'.format(time.time() - cron) From 7ee4efbf4eeb56291dc07b090b720ab3e3ba5a4b Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Tue, 23 Sep 2014 12:00:19 +0200 Subject: [PATCH 32/43] delete old test structure and refactor test to new testenvironment --- test_blackbox.py | 430 +++++++---------------------------------------- 1 file changed, 63 insertions(+), 367 deletions(-) diff --git a/test_blackbox.py b/test_blackbox.py index e3deeb6..be2bdca 100644 --- a/test_blackbox.py +++ b/test_blackbox.py @@ -1,399 +1,95 @@ #!/usr/bin/env python #-*- coding: utf-8 -*- -import subprocess -import time +from systemtest import BlackBoxTest, rand_content +import unittest import os import shutil -import random -import ConfigParser -from passlib.hash import sha256_crypt -import json -import unittest -import filecmp -import hashlib -import string - -import select -import fcntl - -INIT_TIME = 5 -TRY_TIME = 5 -MAX_TRY = 3 - -PROCESS = [] - -TEST_ENV_DIR = os.path.join(os.path.expanduser('~'), 'TestEnv') - -# test user information: -# user: user1@test.it -# istance1: istance sinked with server -# istance2: istance of daemon not sinked with server - -CONF_USERS = { - 'istance1': { - 'username': 'user1@test.it', - 'password': 'password', - 'conf_path': os.path.join(TEST_ENV_DIR, 'istance1/user1@test.it_config'), - 'share_path': os.path.join(TEST_ENV_DIR, 'istance1/user1@test.it_share'), - 'dmn_command': 'cd {}; python {}/client/client_daemon.py'.format( - os.path.join(TEST_ENV_DIR, 'istance1/user1@test.it_config'), - os.path.dirname(os.path.abspath(__file__))) - }, - 'istance2': { - 'username': 'user1@test.it', - 'password': 'password', - 'conf_path': os.path.join(TEST_ENV_DIR, 'istance2/user1@test.it_config'), - 'share_path': os.path.join(TEST_ENV_DIR, 'istance2/user1@test.it_share'), - 'dmn_command': 'cd {}; python {}/client/client_daemon.py'.format( - os.path.join(TEST_ENV_DIR, 'istance2/user1@test.it_config'), - os.path.dirname(os.path.abspath(__file__))) - } -} - -# local server sincronized test directory for user "user1" -SERVER_USER_DATA = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "server", "user_data.json") -SERVER_USER_DIR = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - 'server/user_dirs/', - 'user1@test.it') - -SYNC_TIME = time.time() - - -def non_block_readline(output): - fd = output.fileno() - fl = fcntl.fcntl(fd, fcntl.F_GETFL) - fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) - try: - return output.readline() - except IOError: - pass - - -def check_folder(src_folder, dst_folder): - for root, dirs, files in os.walk(src_folder): - for f in files: - full_src_path = os.path.join(root, f) - rel_path = os.path.relpath(full_src_path, src_folder) - full_dst_path = os.path.join(dst_folder, rel_path) - try: - result = filecmp.cmp(full_src_path, full_dst_path) - except OSError: - result = False - if not result: - print "---" - print "check propagation for: {}".format(rel_path) - print "from: {}".format(full_src_path) - print "to: {}".format(full_dst_path) - return result - return result - - -def check_propagation(first_folder, second_folder): - ''' - check identity betwen two specific folders contenute - ''' - # direct check - if not check_folder(first_folder, second_folder): - return False - - # reverse check - if not check_folder(second_folder, first_folder): - return False - - return True - - -def rand_content(size=4, chars=string.ascii_uppercase + string.digits): - ''' function for random string creation ''' - - return ''.join(random.choice(chars) for _ in range(size)) - - -def create_file(folder): - ''' - function for random file creation in a specific folder. - return the path of file and the md5 of content - ''' - while True: - filename = rand_content() - filepath = os.path.join(folder, filename) - if not os.path.exists(filepath): - content = rand_content(10) - open(filepath, 'w').write(content) - md5 = hashlib.md5(content).hexdigest() - return filepath, md5 - - -def create_folder_tree(main_path, user): - folders = ['mass_copy', 'mass_move', 'mass_delete', 'mass_modify'] - client_snap = {} - svr_filelist = {} - svr_filelist[''] = [ - user, - None, - SYNC_TIME - ] - - for folder in folders: - os.makedirs(os.path.join(main_path, folder)) - svr_filelist[folder] = [ - '/'.join([user, folder]), - None, - SYNC_TIME - ] - for i in range(10): - full_path, md5 = create_file(os.path.join(main_path, folder)) - rel_path = os.path.relpath(full_path, main_path) - if md5 in client_snap: - client_snap[md5].append(rel_path) - else: - client_snap[md5] = [rel_path] - - svr_filelist[rel_path] = [ - '/'.join([user, rel_path]), - md5, - SYNC_TIME - ] - - return client_snap, svr_filelist - -def create_test_environment(): - if os.path.exists(TEST_ENV_DIR): - shutil.rmtree(TEST_ENV_DIR) - if os.path.exists(SERVER_USER_DIR): - shutil.rmtree(SERVER_USER_DIR) +class IntegrationTest(BlackBoxTest): - os.makedirs(TEST_ENV_DIR) - os.makedirs(CONF_USERS['istance1']['conf_path']) - os.makedirs(CONF_USERS['istance1']['share_path']) - os.makedirs(CONF_USERS['istance2']['conf_path']) - os.makedirs(CONF_USERS['istance2']['share_path']) - - client_snap, svr_filelist = create_folder_tree( - CONF_USERS['istance1']['share_path'], - CONF_USERS['istance1']['username']) - shutil.copytree(CONF_USERS['istance1']['share_path'], SERVER_USER_DIR) - - # Create daemon config - port = 6666 - for k, v in CONF_USERS.items(): - daemon_ini = ConfigParser.ConfigParser() - daemon_ini.add_section('cmd') - daemon_ini.set('cmd', 'host', 'localhost') - daemon_ini.set('cmd', 'port', port) - daemon_ini.add_section('daemon_communication') - daemon_ini.set('daemon_communication', 'snapshot_file_path', 'snapshot_file.json') - daemon_ini.set('daemon_communication', 'dir_path', v['share_path']) - daemon_ini.set('daemon_communication', 'server_url', 'localhost') - daemon_ini.set('daemon_communication', 'server_port', 5000) - daemon_ini.set('daemon_communication', 'api_prefix', 'API/v1') - daemon_ini.set('daemon_communication', 'crash_repo_path', os.path.join(v['conf_path'], 'RawBox_crash_report.log')) - daemon_ini.set('daemon_communication', 'stdout_log_level', "DEBUG") - daemon_ini.set('daemon_communication', 'file_log_level', "ERROR") - daemon_ini.add_section('daemon_user_data') - daemon_ini.set('daemon_user_data', 'username', v['username']) - daemon_ini.set('daemon_user_data', 'password', v['password']) - daemon_ini.set('daemon_user_data', 'active', 'True') - - with open(os.path.join(v['conf_path'], "config.ini"), "w") as f: - daemon_ini.write(f) - port += 1 - - #snapshot for sinked instance - with open(os.path.join(CONF_USERS['istance1']['conf_path'], "snapshot_file.json"), "w") as f: - for k, v in client_snap.items(): - v.sort() - snap_list = sorted(list(client_snap.items())) - hashlib.md5(str(snap_list)).hexdigest() - snap = { - "timestamp": SYNC_TIME, - "snapshot": hashlib.md5(str(snap_list)).hexdigest(), - } - json.dump(snap, f) - - # Create server config - svr_conf = {} - svr_conf['users'] = { - CONF_USERS['istance1']['username']: { - "paths": svr_filelist, - "psw": sha256_crypt.encrypt(CONF_USERS['istance1']['password']), - "timestamp": SYNC_TIME, - }, - } - json.dump(svr_conf, open(SERVER_USER_DATA, 'w')) + def test_mass_copy(self): + ist_id = self.env.add_dmn_istance( + credential={'usr': 'test@test.it', 'psw': 'TestPsw1<'}, + svr_rec=True, + dmn_rec=True) + src_path = self.env.add_fld_to_ist(ist_id, 'src_path') + self.env.add_rndfile_to_ist(ist_id, relpath='src_path') + dst_path = os.path.join( + self.env.get_share_path(ist_id), + 'dst_path') + + self.env.start_test_environment() - # Start subprocess - svr_process = subprocess.Popen( - "cd server; python server.py", - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + shutil.copytree( + src_path, + dst_path) - print "server PID: {}".format(svr_process.pid) - time.sleep(2) + self._check_folder() - dmn_process = subprocess.Popen( - CONF_USERS['istance1']['dmn_command'], - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - print "daemon PID: {}".format(dmn_process.pid) + def test_mass_move(self): + ist_id = self.env.add_dmn_istance( + credential={'usr': 'test@test.it', 'psw': 'TestPsw1<'}, + svr_rec=True, + dmn_rec=True) + src_path = self.env.add_fld_to_ist(ist_id, 'src_path') + self.env.add_rndfile_to_ist(ist_id, relpath='src_path') + dst_path = os.path.join( + self.env.get_share_path(ist_id), + 'dst_path') + + self.env.start_test_environment() - time.sleep(INIT_TIME) - return svr_process, dmn_process + shutil.move( + src_path, + dst_path) + self._check_folder() -class BlackBoxTest(unittest.TestCase): + def test_mass_delete(self): + ist_id = self.env.add_dmn_istance( + credential={'usr': 'test@test.it', 'psw': 'TestPsw1<'}, + svr_rec=True, + dmn_rec=True) + del_folder = self.env.add_fld_to_ist(ist_id, 'del_folder') + self.env.add_rndfile_to_ist(ist_id, relpath='del_folder') - @classmethod - def tearDownClass(cls): - for proc in PROCESS: - print '---' - print proc.pid - print proc.poll() - proc.kill() - print proc.poll() + self.env.start_test_environment() - shutil.rmtree(TEST_ENV_DIR) - shutil.rmtree(SERVER_USER_DIR) + shutil.rmtree(del_folder) - def _check_folder(self, shared_path, server_path): - retry = 0 - while True: - print " try: {}".format(retry) - time.sleep(TRY_TIME) - try: - self.assertTrue( - check_propagation( - shared_path, - server_path)) - break - except AssertionError: - if retry >= MAX_TRY: - raise - else: - retry += 1 + self._check_folder() - def test_mass_copy(self): - src_folder = os.path.join( - CONF_USERS['istance1']['share_path'], - 'mass_copy') - dst_folder = os.path.join( - CONF_USERS['istance1']['share_path'], - 'mass_copy_dst') - shutil.copytree( - src_folder, - dst_folder) + def test_mass_create(self): + ist_id = self.env.add_dmn_istance( + credential={'usr': 'test@test.it', 'psw': 'TestPsw1<'}, + svr_rec=True, + dmn_rec=True) - self._check_folder( - CONF_USERS['istance1']['share_path'], - SERVER_USER_DIR) + self.env.start_test_environment() - def test_mass_move(self): - src_folder = os.path.join( - CONF_USERS['istance1']['share_path'], - 'mass_move') - dst_folder = os.path.join( - CONF_USERS['istance1']['share_path'], - 'mass_move_dst') - os.makedirs(dst_folder) - shutil.move( - src_folder, - dst_folder) + self.env.add_rndfile_to_ist(ist_id) - self._check_folder( - CONF_USERS['istance1']['share_path'], - SERVER_USER_DIR) + self._check_folder() - def test_mass_delete(self): - del_folder = os.path.join( - CONF_USERS['istance1']['share_path'], - 'mass_delete') - shutil.rmtree(del_folder) - - self._check_folder( - CONF_USERS['istance1']['share_path'], - SERVER_USER_DIR) + def test_mass_modify(self): + ist_id = self.env.add_dmn_istance( + credential={'usr': 'test@test.it', 'psw': 'TestPsw1<'}, + svr_rec=True, + dmn_rec=True) - def test_mass_create(self): - crt_folder = os.path.join( - CONF_USERS['istance1']['share_path'], - 'mass_create') - os.makedirs(crt_folder) - for i in range(15): - create_file(crt_folder) + self.env.add_rndfile_to_ist(ist_id) - self._check_folder( - CONF_USERS['istance1']['share_path'], - SERVER_USER_DIR) + self.env.start_test_environment() - def test_mass_modify(self): - mod_folder = os.path.join( - CONF_USERS['istance1']['share_path'], - 'mass_modify') - for root, dirs, files in os.walk(mod_folder): + for root, dirs, files in os.walk(self.env.get_share_path(ist_id)): for f in files: - print f filepath = os.path.join(root, f) open(filepath, 'w').write(rand_content(15)) - self._check_folder( - CONF_USERS['istance1']['share_path'], - SERVER_USER_DIR) - - def test_new_client(self): - dmn_process = subprocess.Popen( - CONF_USERS['istance2']['dmn_command'], - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - print "daemon PID: {}".format(dmn_process.pid) - PROCESS.append(dmn_process) - time.sleep(INIT_TIME) - - self._check_folder( - CONF_USERS['istance2']['share_path'], - SERVER_USER_DIR) + self._check_folder() if __name__ == "__main__": - - svr_process, dmn_process = create_test_environment() - PROCESS.append(svr_process) - PROCESS.append(dmn_process) - - test = True - - if test: - unittest.main() - else: - out_list = [] - read_comm = {} - - for proc in PROCESS: - out_list.append(proc.stdout.fileno()) - read_comm[proc.stdout.fileno()] = proc.stdout - out_list.append(proc.stderr.fileno()) - read_comm[proc.stderr.fileno()] = proc.stderr - - for k in read_comm: - print 'fileno: {}'.format(k) - - while True: - try: - r, w, e = select.select(out_list, [], [], 5) - except select.error, err: - raise - - for fd in r: - message = non_block_readline(read_comm.get(fd, None)).strip() - if message != "": - print "read message: {}".format(fd) - print message - print "---" + unittest.main() From fea0bb7f49628915304c51d20f7e3eb901201b5e Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Wed, 24 Sep 2014 11:58:26 +0200 Subject: [PATCH 33/43] fix merge problem --- systemtest/testenvironment.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 1fbebff..d350fd8 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -14,6 +14,7 @@ #import client daemon snapshot manager object from client import DirSnapshotManager +from client import client_daemon import string import random @@ -147,9 +148,7 @@ def create_ist_conf_file(ist_information): daemon_ini.add_section('daemon_communication') daemon_ini.set('daemon_communication', 'snapshot_file_path', 'snapshot_file.json') daemon_ini.set('daemon_communication', 'dir_path', ist_information['share_path']) - daemon_ini.set('daemon_communication', 'server_url', 'localhost') - daemon_ini.set('daemon_communication', 'server_port', 5000) - daemon_ini.set('daemon_communication', 'api_prefix', 'API/v1') + daemon_ini.set('daemon_communication', 'server_url', 'http://localhost:5000/API/v1') daemon_ini.set('daemon_communication', 'crash_repo_path', os.path.join(ist_information['conf_path'], 'RawBox_crash_report.log')) daemon_ini.set('daemon_communication', 'stdout_log_level', "DEBUG") daemon_ini.set('daemon_communication', 'file_log_level', "ERROR") @@ -162,8 +161,10 @@ def create_ist_conf_file(ist_information): daemon_ini.write(f) -def create_spanshot_file(share_path, timestamp, snap_path, synced=True): - open(snap_path, 'w').write('{}') +def create_spanshot_file(share_path, timestamp, snap_path, snap_sync, file_sync): + client_daemon.CONFIG_DIR_PATH = share_path + json.dump({"timestamp": 0, "snapshot": ""}, open(snap_path, 'w')) + # open(snap_path, 'w').write('{}') if synced: snap_manager = DirSnapshotManager(snap_path, share_path) From dad55c8da232ad533fa0ced315588e7550740545 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Wed, 24 Sep 2014 12:00:12 +0200 Subject: [PATCH 34/43] add parameter for test_new_client issue --- systemtest/testenvironment.py | 37 +++++++++++++++++++++-------------- test_blackbox.py | 12 ++++++++++++ 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index d350fd8..3b177dc 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -105,8 +105,9 @@ def update_srv_userdata_adt(ist_information, svr_datastorage, sync_time, svr_usr parameters: ist_information - dict with the daemon istance information as EnvironmentManager's dmn_istance_list - svr_datastorage - server storage reference + svr_datastorage - server storage datafile reference sync_time - timestamp of synchronization + svr_usr_dir - user file directory ''' try: svr_conf = json.load(open(svr_datastorage, 'r')) @@ -126,7 +127,7 @@ def update_srv_userdata_adt(ist_information, svr_datastorage, sync_time, svr_usr "psw": sha256_crypt.encrypt(ist_information['psw']), "timestamp": sync_time, } - if 'svr_filelist' in ist_information: + if 'svr_filelist' in ist_information and ist_information['file_svr_sync']: shutil.copytree( ist_information['share_path'], os.path.join(svr_usr_dir, ist_information['usr'])) @@ -166,7 +167,10 @@ def create_spanshot_file(share_path, timestamp, snap_path, snap_sync, file_sync) json.dump({"timestamp": 0, "snapshot": ""}, open(snap_path, 'w')) # open(snap_path, 'w').write('{}') - if synced: + if not file_sync: + shutil.rmtree(share_path) + os.makedirs(share_path) + if snap_sync: snap_manager = DirSnapshotManager(snap_path, share_path) snap_manager.save_snapshot(timestamp) @@ -281,13 +285,6 @@ def _ist_propagation(self, ist_id): # istance's config file creation create_ist_conf_file(self.dmn_istance_list[ist_id]) - # istance's stapshot file creation - create_spanshot_file( - self.dmn_istance_list[ist_id]['share_path'], - self.sync_time, - os.path.join(self.dmn_istance_list[ist_id]['conf_path'], 'snapshot_file.json'), - self.dmn_istance_list[ist_id]['self_sync']) - # propagation in server user_data.json if self.dmn_istance_list[ist_id]['svr_rec']: update_srv_userdata_adt( @@ -296,6 +293,14 @@ def _ist_propagation(self, ist_id): self.sync_time, self.svr_usr_dir) + # istance's stapshot file creation + create_spanshot_file( + self.dmn_istance_list[ist_id]['share_path'], + self.sync_time, + os.path.join(self.dmn_istance_list[ist_id]['conf_path'], 'snapshot_file.json'), + self.dmn_istance_list[ist_id]['snap_sync'], + self.dmn_istance_list[ist_id]['file_dmn_sync']) + def start_test_environment(self): '''Start all enviromnent server and istance process ''' @@ -412,18 +417,20 @@ def add_dmn_istance(self, ist_id=None, credential=None, svr_rec=False, dmn_rec=F return ist_id - def sync_dmn_share(self, ist_id, self_sync=True, server_sync=True): + def sync_dmn_share(self, ist_id, file_dmn_sync=True, file_svr_sync=True, snap_sync=True): ''' synchronize istance share folder in the configuration file. Attributes: ist_id - istance id - self_sync - boolean. if true it will synchronize the snapshot file + file_dmn_sync - boolean. if true it will persist the file on daemon share dir + file_svr_sync - boolean. if true it will synchronize the server user data default True - server_sync - boolean. if true it will synchronize the server user data + snap_sync - boolean. if true it will synchronize the snapshot file default True ''' - self.dmn_istance_list[ist_id]['self_sync'] = self_sync - self.dmn_istance_list[ist_id]['server_sync'] = server_sync + self.dmn_istance_list[ist_id]['file_dmn_sync'] = file_dmn_sync + self.dmn_istance_list[ist_id]['file_svr_sync'] = file_svr_sync + self.dmn_istance_list[ist_id]['snap_sync'] = snap_sync def add_fld_to_ist(self, ist_id, folder): ''' diff --git a/test_blackbox.py b/test_blackbox.py index be2bdca..86c40c2 100644 --- a/test_blackbox.py +++ b/test_blackbox.py @@ -90,6 +90,18 @@ def test_mass_modify(self): self._check_folder() + def test_new_daemon(self): + ist_id = self.env.add_dmn_istance( + credential={'usr': 'test@test.it', 'psw': 'TestPsw1<'}, + svr_rec=True, + dmn_rec=False) + + self.env.add_rndfile_to_ist(ist_id) + self.env.sync_dmn_share(ist_id, False, True, False) + + self.env.start_test_environment() + self._check_folder() + if __name__ == "__main__": unittest.main() From 36a231b8320bd66f51741a1a3705d70412afca72 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Wed, 24 Sep 2014 14:55:54 +0200 Subject: [PATCH 35/43] FIX client_daemon download system --- client/client_daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/client_daemon.py b/client/client_daemon.py index 04b31e8..992083c 100644 --- a/client/client_daemon.py +++ b/client/client_daemon.py @@ -152,7 +152,7 @@ def download_file(self, dst_path): local_path = get_abspath(dst_path) if r.status_code == requests.codes.ok: - return local_path, r.text + return local_path, r.content else: return False, False From 9efb6f34cf303c8cbb7f84bc2851edb6264680d1 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Wed, 24 Sep 2014 14:56:34 +0200 Subject: [PATCH 36/43] revert test refuse correction --- client/test_client_daemon.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/client/test_client_daemon.py b/client/test_client_daemon.py index bbefb25..a25c4b2 100644 --- a/client/test_client_daemon.py +++ b/client/test_client_daemon.py @@ -477,7 +477,7 @@ def test_download(self): #check if methods are equal self.assertEqual(method, 'GET') #check response's body - self.assertEqual(response[1], [{"title": "Test"}]) + self.assertEqual(response[1], u'[{"title": "Test"}]') #Case: server bad request def _try_request(self, *args, **kwargs): @@ -730,10 +730,9 @@ def update_snapshot_upload(self, body): if not os.path.exists(self.client_path): os.makedirs(self.client_path) httpretty.enable() - test_json = json.dumps('stringa_test') httpretty.register_uri(httpretty.GET, 'http://localhost/api/v1/files/{}'.format(self.filename), - body='"this is a test"', - content_type='json') + body='this is a test', + content_type='text/plain') self.snapshot_manager = DirSnapshotManager() self.server_com = ServerCommunicator( server_url='http://localhost/api/v1', From f77f6e799e835f8de74db8c28149a57e8374b229 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Wed, 24 Sep 2014 14:57:12 +0200 Subject: [PATCH 37/43] fix server folder creation problem --- systemtest/testenvironment.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 3b177dc..b555191 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -115,6 +115,10 @@ def update_srv_userdata_adt(ist_information, svr_datastorage, sync_time, svr_usr svr_conf = { "users": {} } + try: + os.makedirs(os.path.join(svr_usr_dir, ist_information['usr'])) + except OSError: + pass svr_conf['users'][ist_information['usr']] = { "paths": { @@ -128,13 +132,13 @@ def update_srv_userdata_adt(ist_information, svr_datastorage, sync_time, svr_usr "timestamp": sync_time, } if 'svr_filelist' in ist_information and ist_information['file_svr_sync']: + shutil.rmtree(os.path.join(svr_usr_dir, ist_information['usr'])) shutil.copytree( ist_information['share_path'], os.path.join(svr_usr_dir, ist_information['usr'])) svr_conf['users'][ist_information['usr']]['paths'].update( ist_information['svr_filelist']) - else: - os.makedirs(os.path.join(svr_usr_dir, ist_information['usr'])) + json.dump(svr_conf, open(svr_datastorage, 'w')) From 763adcaf5c12d5e73c5ade572714ac2c6bf05ef9 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Fri, 28 Nov 2014 15:13:50 +0100 Subject: [PATCH 38/43] change variable name to prevent nomenclature mistake --- systemtest/testenvironment.py | 42 +++++++++++++++++------------------ test_blackbox.py | 6 ++--- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index b555191..dec8105 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -134,7 +134,7 @@ def update_srv_userdata_adt(ist_information, svr_datastorage, sync_time, svr_usr if 'svr_filelist' in ist_information and ist_information['file_svr_sync']: shutil.rmtree(os.path.join(svr_usr_dir, ist_information['usr'])) shutil.copytree( - ist_information['share_path'], + ist_information['rawbox_dir'], os.path.join(svr_usr_dir, ist_information['usr'])) svr_conf['users'][ist_information['usr']]['paths'].update( ist_information['svr_filelist']) @@ -152,7 +152,7 @@ def create_ist_conf_file(ist_information): daemon_ini.set('cmd', 'port', ist_information['dmn_port']) daemon_ini.add_section('daemon_communication') daemon_ini.set('daemon_communication', 'snapshot_file_path', 'snapshot_file.json') - daemon_ini.set('daemon_communication', 'dir_path', ist_information['share_path']) + daemon_ini.set('daemon_communication', 'dir_path', ist_information['rawbox_dir']) daemon_ini.set('daemon_communication', 'server_url', 'http://localhost:5000/API/v1') daemon_ini.set('daemon_communication', 'crash_repo_path', os.path.join(ist_information['conf_path'], 'RawBox_crash_report.log')) daemon_ini.set('daemon_communication', 'stdout_log_level', "DEBUG") @@ -166,16 +166,16 @@ def create_ist_conf_file(ist_information): daemon_ini.write(f) -def create_spanshot_file(share_path, timestamp, snap_path, snap_sync, file_sync): - client_daemon.CONFIG_DIR_PATH = share_path +def create_spanshot_file(rawbox_dir, timestamp, snap_path, snap_sync, file_sync): + client_daemon.CONFIG_DIR_PATH = rawbox_dir json.dump({"timestamp": 0, "snapshot": ""}, open(snap_path, 'w')) # open(snap_path, 'w').write('{}') if not file_sync: - shutil.rmtree(share_path) - os.makedirs(share_path) + shutil.rmtree(rawbox_dir) + os.makedirs(rawbox_dir) if snap_sync: - snap_manager = DirSnapshotManager(snap_path, share_path) + snap_manager = DirSnapshotManager(snap_path, rawbox_dir) snap_manager.save_snapshot(timestamp) @@ -224,7 +224,7 @@ def __init__( -config.ini -snapshot.json -RawBox_crash_report.log - -share- + -rawbox- -file_to_share Server's folder structure: |root @@ -299,7 +299,7 @@ def _ist_propagation(self, ist_id): # istance's stapshot file creation create_spanshot_file( - self.dmn_istance_list[ist_id]['share_path'], + self.dmn_istance_list[ist_id]['rawbox_dir'], self.sync_time, os.path.join(self.dmn_istance_list[ist_id]['conf_path'], 'snapshot_file.json'), self.dmn_istance_list[ist_id]['snap_sync'], @@ -335,8 +335,8 @@ def stop_test_environment(self): self._stop_serverproc() print 'Stopped' - def get_share_path(self, ist_id): - return self.dmn_istance_list[ist_id]['share_path'] + def get_rawbox_dir(self, ist_id): + return self.dmn_istance_list[ist_id]['rawbox_dir'] def flush(self): ''' @@ -404,20 +404,20 @@ def add_dmn_istance(self, ist_id=None, credential=None, svr_rec=False, dmn_rec=F istance_path = os.path.join(self.dmn_test_dir, ist_id) conf_path = os.path.join(istance_path, 'config') - share_path = os.path.join(istance_path, 'share') + rawbox_dir = os.path.join(istance_path, 'rawbox') self.dmn_istance_list[ist_id]['root_path'] = istance_path self.dmn_istance_list[ist_id]['conf_path'] = conf_path - self.dmn_istance_list[ist_id]['share_path'] = share_path + self.dmn_istance_list[ist_id]['rawbox_dir'] = rawbox_dir if os.path.exists(istance_path): shutil.rmtree(istance_path) if os.path.exists(conf_path): shutil.rmtree(conf_path) - if os.path.exists(share_path): - shutil.rmtree(share_path) + if os.path.exists(rawbox_dir): + shutil.rmtree(rawbox_dir) os.makedirs(istance_path) os.makedirs(conf_path) - os.makedirs(share_path) + os.makedirs(rawbox_dir) return ist_id @@ -441,7 +441,7 @@ def add_fld_to_ist(self, ist_id, folder): add new folder in istance's share directory identified by ist_id ''' path = os.path.join( - self.dmn_istance_list[ist_id]['share_path'], + self.dmn_istance_list[ist_id]['rawbox_dir'], folder) if not os.path.exists(path): os.makedirs(path) @@ -460,10 +460,10 @@ def add_rndfile_to_ist(self, ist_id, num_file=10, relpath=None): ''' if relpath: dst_path = os.path.join( - self.dmn_istance_list[ist_id]['share_path'], + self.dmn_istance_list[ist_id]['rawbox_dir'], relpath) else: - dst_path = self.dmn_istance_list[ist_id]['share_path'] + dst_path = self.dmn_istance_list[ist_id]['rawbox_dir'] if 'svr_filelist' not in self.dmn_istance_list[ist_id]: self.dmn_istance_list[ist_id]['svr_filelist'] = {} @@ -472,7 +472,7 @@ def add_rndfile_to_ist(self, ist_id, num_file=10, relpath=None): filename, full_path, md5 = create_file(dst_path) rel_path = os.path.relpath( full_path, - self.dmn_istance_list[ist_id]['share_path']) + self.dmn_istance_list[ist_id]['rawbox_dir']) if filename not in self.dmn_istance_list[ist_id]['svr_filelist']: self.dmn_istance_list[ist_id]['svr_filelist'][rel_path] = [] @@ -548,7 +548,7 @@ def _check_folder(self): try: self.assertTrue( check_propagation( - ist['share_path'], + ist['rawbox_dir'], os.path.join(self.env.svr_usr_dir, ist['usr']))) break except AssertionError: diff --git a/test_blackbox.py b/test_blackbox.py index 86c40c2..d583941 100644 --- a/test_blackbox.py +++ b/test_blackbox.py @@ -17,7 +17,7 @@ def test_mass_copy(self): src_path = self.env.add_fld_to_ist(ist_id, 'src_path') self.env.add_rndfile_to_ist(ist_id, relpath='src_path') dst_path = os.path.join( - self.env.get_share_path(ist_id), + self.env.get_rawbox_dir(ist_id), 'dst_path') self.env.start_test_environment() @@ -36,7 +36,7 @@ def test_mass_move(self): src_path = self.env.add_fld_to_ist(ist_id, 'src_path') self.env.add_rndfile_to_ist(ist_id, relpath='src_path') dst_path = os.path.join( - self.env.get_share_path(ist_id), + self.env.get_rawbox_dir(ist_id), 'dst_path') self.env.start_test_environment() @@ -83,7 +83,7 @@ def test_mass_modify(self): self.env.start_test_environment() - for root, dirs, files in os.walk(self.env.get_share_path(ist_id)): + for root, dirs, files in os.walk(self.env.get_rawbox_dir(ist_id)): for f in files: filepath = os.path.join(root, f) open(filepath, 'w').write(rand_content(15)) From b2a016c0caa62fc86f5fa673c3a8cafeee9e7a00 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Fri, 28 Nov 2014 17:54:00 +0100 Subject: [PATCH 39/43] fix adapter form --- systemtest/testenvironment.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index dec8105..49a04ef 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -4,7 +4,6 @@ import os import json import time -import re import ConfigParser import shutil from passlib.hash import sha256_crypt @@ -55,12 +54,10 @@ def check_password(password): return a Boolean ''' - if PasswordChecker(password) == password: - return True - else: - return False + return PasswordChecker(password) == password +from client import EMAIL_REGEX def check_username(username): ''' username checker adapter @@ -69,8 +66,7 @@ def check_username(username): retunr a Boolean ''' - email_regex = re.compile('[^@]+@[^@]+\.[^@]+') - return email_regex.match(username) + return EMAIL_REGEX.match(username) def start_proc(command): From 666ecfe945b9408e6f4b8a9aaf4a40f1f5c5ba56 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Fri, 28 Nov 2014 17:54:11 +0100 Subject: [PATCH 40/43] fix some typo error --- systemtest/testenvironment.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 49a04ef..6ecd032 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -64,7 +64,7 @@ def check_username(username): parameters: username - string - retunr a Boolean + return a Boolean ''' return EMAIL_REGEX.match(username) @@ -199,8 +199,8 @@ def __init__( ): ''' Create a folder tree for the execution of system test. - patameters: - dmn_src_path - daemnon and cmd_manager source absolute path + parameters: + dmn_src_path - daemon and cmd_manager source absolute path default - os.path.join(os.getcwd(),'/client/') svr_src_path - server source absolute path default - os.path.join(os.getcwd(),'/server/') @@ -226,7 +226,7 @@ def __init__( |root -svr_usr_dir- -username- - -file_to_shar + -file_to_share ''' self.dmn_src_path = dmn_src_path self.svr_src_path = svr_src_path From 3efe9c08d1d0454501d9fd40d3cbb72fa9f9b80b Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 1 Dec 2014 10:57:49 +0100 Subject: [PATCH 41/43] some code optimization --- systemtest/testenvironment.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 6ecd032..4021afb 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -441,9 +441,7 @@ def add_fld_to_ist(self, ist_id, folder): folder) if not os.path.exists(path): os.makedirs(path) - if 'svr_filelist' not in self.dmn_istance_list[ist_id]: - self.dmn_istance_list[ist_id]['svr_filelist'] = {} - self.dmn_istance_list[ist_id]['svr_filelist'][folder] = [ + self.dmn_istance_list[ist_id].setdefault('svr_filelist', {})[folder] = [ os.path.join(self.dmn_istance_list[ist_id]['usr'], folder), None, self.sync_time @@ -464,20 +462,17 @@ def add_rndfile_to_ist(self, ist_id, num_file=10, relpath=None): if 'svr_filelist' not in self.dmn_istance_list[ist_id]: self.dmn_istance_list[ist_id]['svr_filelist'] = {} - for e_file in range(num_file): + for e_file in xrange(num_file): filename, full_path, md5 = create_file(dst_path) rel_path = os.path.relpath( full_path, self.dmn_istance_list[ist_id]['rawbox_dir']) - if filename not in self.dmn_istance_list[ist_id]['svr_filelist']: - self.dmn_istance_list[ist_id]['svr_filelist'][rel_path] = [] - - self.dmn_istance_list[ist_id]['svr_filelist'][rel_path].append( - os.path.join(self.dmn_istance_list[ist_id]['usr'], rel_path)) - self.dmn_istance_list[ist_id]['svr_filelist'][rel_path].append(md5) - self.dmn_istance_list[ist_id]['svr_filelist'][rel_path].append( - self.sync_time) + self.dmn_istance_list[ist_id]['svr_filelist'][rel_path] = [ + os.path.join(self.dmn_istance_list[ist_id]['usr'], rel_path), + md5, + self.sync_time + ] def check_folder(src_folder, dst_folder): From b6b2b8b33d0aaf754cebb28cefeba88d7feab033 Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 1 Dec 2014 10:59:34 +0100 Subject: [PATCH 42/43] verbose use of function in test --- test_blackbox.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test_blackbox.py b/test_blackbox.py index d583941..0872708 100644 --- a/test_blackbox.py +++ b/test_blackbox.py @@ -97,7 +97,11 @@ def test_new_daemon(self): dmn_rec=False) self.env.add_rndfile_to_ist(ist_id) - self.env.sync_dmn_share(ist_id, False, True, False) + self.env.sync_dmn_share( + ist_id, + file_dmn_sync=False, + file_svr_sync=True, + snap_sync=False) self.env.start_test_environment() self._check_folder() From 9f4e3f8d8ccda272e8bcf4447b97dea9cd632caa Mon Sep 17 00:00:00 2001 From: Federico Della Rovere Date: Mon, 1 Dec 2014 11:47:31 +0100 Subject: [PATCH 43/43] refactor instance inizialization --- systemtest/testenvironment.py | 53 ++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/systemtest/testenvironment.py b/systemtest/testenvironment.py index 4021afb..d4b520c 100644 --- a/systemtest/testenvironment.py +++ b/systemtest/testenvironment.py @@ -385,35 +385,42 @@ def add_dmn_istance(self, ist_id=None, credential=None, svr_rec=False, dmn_rec=F if svr_rec: if check_password(credential['psw']) and check_username(credential['usr']): - self.dmn_istance_list[ist_id]['usr'] = credential['usr'] - self.dmn_istance_list[ist_id]['psw'] = credential['psw'] - self.dmn_istance_list[ist_id]['dmn_rec'] = dmn_rec + user_dict = { + 'usr': credential['usr'], + 'psw': credential['psw'], + 'dmn_rec': dmn_rec, + 'svr_rec': svr_rec, + } else: raise InputError('credential not valid') else: - self.dmn_istance_list[ist_id]['usr'] = None - self.dmn_istance_list[ist_id]['psw'] = None - self.dmn_istance_list[ist_id]['dmn_rec'] = False - self.dmn_istance_list[ist_id]['svr_rec'] = svr_rec + user_dict = { + 'usr': None, + 'psw': None, + 'dmn_rec': False, + 'svr_rec': svr_rec, + } + self.dmn_istance_list[ist_id].update(user_dict) self.sync_dmn_share(ist_id) - istance_path = os.path.join(self.dmn_test_dir, ist_id) - conf_path = os.path.join(istance_path, 'config') - rawbox_dir = os.path.join(istance_path, 'rawbox') - self.dmn_istance_list[ist_id]['root_path'] = istance_path - self.dmn_istance_list[ist_id]['conf_path'] = conf_path - self.dmn_istance_list[ist_id]['rawbox_dir'] = rawbox_dir - - if os.path.exists(istance_path): - shutil.rmtree(istance_path) - if os.path.exists(conf_path): - shutil.rmtree(conf_path) - if os.path.exists(rawbox_dir): - shutil.rmtree(rawbox_dir) - os.makedirs(istance_path) - os.makedirs(conf_path) - os.makedirs(rawbox_dir) + instance_path = os.path.join(self.dmn_test_dir, ist_id) + path_dict = { + 'root_path': instance_path, + 'conf_path': os.path.join(instance_path, 'config'), + 'rawbox_dir': os.path.join(instance_path, 'rawbox'), + } + self.dmn_istance_list[ist_id].update(path_dict) + + if os.path.exists(path_dict['root_path']): + shutil.rmtree(path_dict['root_path']) + if os.path.exists(path_dict['conf_path']): + shutil.rmtree(path_dict['conf_path']) + if os.path.exists(path_dict['rawbox_dir']): + shutil.rmtree(path_dict['rawbox_dir']) + os.makedirs(path_dict['root_path']) + os.makedirs(path_dict['conf_path']) + os.makedirs(path_dict['rawbox_dir']) return ist_id