diff --git a/.gitignore b/.gitignore index 0447b8b..d1bf7fa 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +.idea/ # PyInstaller # Usually these files are written by a python script from a template diff --git a/api/dev.Dockerfile b/api/dev.Dockerfile new file mode 100644 index 0000000..e702c87 --- /dev/null +++ b/api/dev.Dockerfile @@ -0,0 +1,22 @@ +FROM dannixon/savu:latest + +# Install additional Python dependencies +ADD requirements.txt \ + /requirements.txt + +# Install project requirements, AND install pydevd - for remote debugging with PyCharm +RUN /miniconda/bin/pip install -r /requirements.txt && \ + rm /requirements.txt && /miniconda/bin/pip install pydevd + +# Copy API application configuration +ADD ./config/dls.json \ + /hebi_config.json + +# Expose API port +EXPOSE 5000 + +# Run server +CMD ["/webservice/run.sh"] + +# Note: application files not added, they must be mounted +# via `docker run` via `-v/--mount /development/source:/webservice` \ No newline at end of file diff --git a/api/readme.md b/api/readme.md index b80a681..79e3dfa 100644 --- a/api/readme.md +++ b/api/readme.md @@ -16,13 +16,13 @@ from each version of the Savu image that is desired. ### Plugin - List all plugins - `GET /api/plugin` + `GET /api/plugins` - Search plugins by name - `GET /api/plugin?q=tomo` + `GET /api/plugins?q=tomo` - List plugin details - `GET /api/plugin/TomopyRecon` + `GET /api/plugins/TomopyRecon` ### Process list diff --git a/api/webservice/const.py b/api/webservice/const.py index 9280b56..cabaa40 100644 --- a/api/webservice/const.py +++ b/api/webservice/const.py @@ -15,6 +15,7 @@ KEY_PLUGINS = 'plugins' KEY_QUERY = 'q' KEY_QUEUE_ID = 'queue' +KEY_DETAILS = 'details' WS_NAMESPACE_JOB_STATUS = '/job_status' diff --git a/api/webservice/server.py b/api/webservice/server.py index ebef00c..c215210 100644 --- a/api/webservice/server.py +++ b/api/webservice/server.py @@ -1,26 +1,25 @@ -import glob import os import os.path -from flask import (Flask, jsonify, request, abort, make_response, send_file) +import json_tricks +import savu.plugins.utils as pu +import voluptuous +from flask import (Flask, jsonify, request, abort, send_file) from flask.json import JSONEncoder from flask_api import status from flask_cors import CORS -from flask_socketio import SocketIO, emit, join_room, leave_room +from flask_socketio import SocketIO, join_room, leave_room from fuzzywuzzy import fuzz -import json_tricks -import voluptuous - -import savu.plugins.utils as pu from scripts.config_generator.content import Content +import const +import urls +import validation +from execution import NoSuchJobError from utils import (plugin_to_dict, plugin_list_entry_to_dict, is_file_a_data_file, is_file_a_process_list, validate_file, - to_bool, create_process_list_from_user_data, + create_process_list_from_user_data, find_files_recursive) -from execution import NoSuchJobError -import const -import validation class BetterJsonEncoder(JSONEncoder): @@ -37,7 +36,7 @@ def default(self, o): def setup_runners(): import importlib for queue_name, runner in app.config[const.CONFIG_NAMESPACE_SAVU][ - const.CONFIG_KEY_JOB_RUNNERS].iteritems(): + const.CONFIG_KEY_JOB_RUNNERS].iteritems(): # Create an instance of the job runner m = importlib.import_module(runner[const.CONFIG_KEY_RUNNER_MODULE]) c = getattr(m, runner[const.CONFIG_KEY_RUNNER_CLASS]) @@ -59,8 +58,7 @@ def send_updates_thread_fun(qn, runner): def teardown_runners(): - for _, v in app.config[const.CONFIG_NAMESPACE_SAVU][ - const.CONFIG_KEY_JOB_RUNNERS]: + for _, v in app.config[const.CONFIG_NAMESPACE_SAVU][const.CONFIG_KEY_JOB_RUNNERS]: v[const.CONFIG_KEY_RUNNER_INSTANCE].close() @@ -69,34 +67,57 @@ def validate_config(): app.config[const.CONFIG_NAMESPACE_SAVU]) -@app.route('/plugin') +@app.route(urls.PLUGINS) def query_plugin_list(): query = request.args.get(const.KEY_QUERY) + append_details = request.args.get(const.KEY_DETAILS, default=False) + if query: query = query.lower() - plugin_names = [k for k, v in pu.plugins.iteritems() \ + plugin_names = [k for k, v in pu.plugins.iteritems() if fuzz.partial_ratio(k.lower(), query) > 75] else: - plugin_names = [k for k, v in pu.plugins.iteritems()] + if append_details: + plugin_names = {} + for k, v in pu.plugins.viewitems(): + plugin_names[k] = _get_plugin_info(k) + + validation.query_plugin_list_with_details_schema(plugin_names) + else: + plugin_names = [k for k, v in pu.plugins.iteritems()] + validation.query_plugin_list_schema(plugin_names) - validation.query_plugin_list_schema(plugin_names) return jsonify(plugin_names) -@app.route('/plugin/') +@app.route('{}/'.format(urls.PLUGINS)) def get_plugin_info(name): + """ + Returns the plugin info in JSON + :param name: + :return: + """ if name not in pu.plugins: abort(status.HTTP_404_NOT_FOUND) + data = _get_plugin_info(name) + + validation.get_plugin_info_schema(data) + return jsonify(data) + + +def _get_plugin_info(name): + """ + Returns the plugin info as a Dict + :param name: + :return: + """ # Create plugin instance with default parameter values p = pu.plugins[name]() p._populate_default_parameters() - data = plugin_to_dict(name, p) - - validation.get_plugin_info_schema(data) - return jsonify(data) + return data @app.route('/process_list') @@ -256,8 +277,8 @@ def jobs_queue_submit(queue): # Start job job = app.config[const.CONFIG_NAMESPACE_SAVU][ const.CONFIG_KEY_JOB_RUNNERS][queue][ - const.CONFIG_KEY_RUNNER_INSTANCE].start_job(dataset, process_list, - output) + const.CONFIG_KEY_RUNNER_INSTANCE].start_job(dataset, process_list, + output) return jobs_queue_info(queue, job) diff --git a/api/webservice/server_test.py b/api/webservice/server_test.py index 3b1f811..256680b 100644 --- a/api/webservice/server_test.py +++ b/api/webservice/server_test.py @@ -1,7 +1,8 @@ -import unittest import json +import unittest import server +import urls from utils import populate_plugins @@ -24,25 +25,32 @@ def setUp(self): }, } - def test_get_plugin(self): - rv = self.app.get('/plugin') + def test_get_plugins(self): + rv = self.app.get(urls.PLUGINS) data = json.loads(rv.data) self.assertTrue(isinstance(data, list)) self.assertGreater(len(data), 0) + def test_get_plugins_with_details(self): + rv = self.app.get("{}?details=true".format(urls.PLUGINS)) + data = json.loads(rv.data) + + self.assertTrue(isinstance(data, dict)) + self.assertGreater(len(data.keys()), 0) + def test_get_plugin_info_no_citation(self): - rv = self.app.get('/plugin/Dezinger') + rv = self.app.get('{}/Dezinger'.format(urls.PLUGINS)) data = json.loads(rv.data) self.assertEqual(len(data['citation']), 0) def test_get_plugin_info_1_citation(self): - rv = self.app.get('/plugin/TomopyRecon') + rv = self.app.get('{}/TomopyRecon'.format(urls.PLUGINS)) data = json.loads(rv.data) self.assertEqual(len(data['citation']), 1) def test_get_plugin_info_multiple_citation(self): - rv = self.app.get('/plugin/AstraReconGpu') + rv = self.app.get('{}/AstraReconGpu'.format(urls.PLUGINS)) data = json.loads(rv.data) self.assertEqual(len(data['citation']), 3) diff --git a/api/webservice/urls.py b/api/webservice/urls.py new file mode 100644 index 0000000..cee5c17 --- /dev/null +++ b/api/webservice/urls.py @@ -0,0 +1 @@ +PLUGINS = "/plugins" diff --git a/api/webservice/validation.py b/api/webservice/validation.py index f7a2fe8..e446618 100644 --- a/api/webservice/validation.py +++ b/api/webservice/validation.py @@ -1,7 +1,9 @@ -from voluptuous import Schema, Required, Optional, All, Any, Length, Number, Extra +from voluptuous import Schema, Required, Optional, All, Any, Length, Extra, ALLOW_EXTRA _string = Any(str, unicode) +_bool = Any(bool, unicode) + _non_empty_string = All(_string, Length(min=1)) _parameter_value = _string @@ -24,6 +26,7 @@ Optional('type'): _non_empty_string, Required('is_user'): bool, Required('is_hidden'): bool, + Required('value'): Any() }) _plugin_basic = { @@ -62,8 +65,6 @@ } }) -query_plugin_list_schema = Schema([_non_empty_string]) - get_plugin_info_schema = Schema({ Required('name'): _non_empty_string, Required('info'): _string, @@ -73,6 +74,12 @@ Required('parameters'): [_parameter_full], }) +query_plugin_list_with_details_schema = Schema({ + Required(_non_empty_string): get_plugin_info_schema +}, extra=ALLOW_EXTRA) + +query_plugin_list_schema = Schema([_non_empty_string], extra=ALLOW_EXTRA) + filename_listing_schema = Schema({ Required('path'): _non_empty_string, Required('files'): [_non_empty_string], diff --git a/web/config/www/src/api_savu.js b/web/config/www/src/api_savu.js index 2851db8..1c575f1 100644 --- a/web/config/www/src/api_savu.js +++ b/web/config/www/src/api_savu.js @@ -1,13 +1,13 @@ function getAvailablePlugins(callback, error) { - jsonGet("/api/plugin", callback, error); + jsonGet("/api/plugins", callback, error); } function searchAvailablePlugins(query, callback, error) { - jsonGet("/api/plugin?q=" + query, callback, error); + jsonGet("/api/plugins?q=" + query, callback, error); } function getPluginDetails(pluginName, callback, error) { - jsonGet("/api/plugin/" + pluginName, callback, error); + jsonGet("/api/plugins/" + pluginName, callback, error); } function getAvailableProcessLists(searchPath, callback, error) {