From a0b49b0ff7dcd25d7085a07ac13fd02ed63992f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?onur=20gu=CC=88ngo=CC=88r?= Date: Mon, 22 Jun 2020 23:35:10 +0300 Subject: [PATCH 1/9] =?UTF-8?q?spellchecker=20art=C4=B1k=20get=20ile=20de?= =?UTF-8?q?=20=C3=A7al=C4=B1=C5=9F=C4=B1yor.=20ikinci=20bir=20sayfaya=20(d?= =?UTF-8?q?eda/cevap)=20ihtiya=C3=A7=20yok.=20url=20de=20g=C3=B6steriyoruz?= =?UTF-8?q?.=20oradan=20kopyalama=20=C3=B6zelli=C4=9Fi=20de=20koyal=C4=B1m?= =?UTF-8?q?.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../spellchecker/spellchecker_answer.html | 39 ------ .../spellchecker/spellchecker_query.html | 49 +++++-- game_project/spellchecker/urls.py | 1 - game_project/spellchecker/views.py | 132 ++++++------------ 4 files changed, 86 insertions(+), 135 deletions(-) delete mode 100644 game_project/spellchecker/templates/spellchecker/spellchecker_answer.html diff --git a/game_project/spellchecker/templates/spellchecker/spellchecker_answer.html b/game_project/spellchecker/templates/spellchecker/spellchecker_answer.html deleted file mode 100644 index 94df40d..0000000 --- a/game_project/spellchecker/templates/spellchecker/spellchecker_answer.html +++ /dev/null @@ -1,39 +0,0 @@ -{% extends 'game/base.html' %} -{% load static %} -{% block content %} - -
- -

-de/-da Yazım Hatası Bulucu

- -
-
- -
- - {% for entry in labeled_words %} - - {% if entry.label == True %} - - {{entry.word}} - - {% else %} - - {{entry.word}} - - {% endif %} - - {% endfor %} - -
- -
- -
- - - Başka bir cümle dene -
-
- -{% endblock content %} diff --git a/game_project/spellchecker/templates/spellchecker/spellchecker_query.html b/game_project/spellchecker/templates/spellchecker/spellchecker_query.html index effd7c0..601af47 100644 --- a/game_project/spellchecker/templates/spellchecker/spellchecker_query.html +++ b/game_project/spellchecker/templates/spellchecker/spellchecker_query.html @@ -10,22 +10,54 @@

-de/-da Yazım Hatası Bulucu



-
- {% csrf_token %} + {% if labeled_words %} +
- + {% for entry in labeled_words %} - + {% if entry.label == True %} -
-
+ {{entry.word}} -

- + {% else %} + {{entry.word}} + + {% endif %} + + {% endfor %} + +
+ +
+
+ Başka bir cümle dene +
+
+ + + {% else %} + +
+ {% csrf_token %} + + + + + +
+
+ +

+
+ + {% endif %} + {% if not labeled_words %}
@@ -37,6 +69,7 @@

-de/-da Yazım Hatası Bulucu

+ {% endif %} + +{% endblock content %} \ No newline at end of file diff --git a/game_project/spellchecker/urls.py b/game_project/spellchecker/urls.py index 78aec6b..08c8290 100644 --- a/game_project/spellchecker/urls.py +++ b/game_project/spellchecker/urls.py @@ -2,9 +2,9 @@ from . import views - urlpatterns = [ - - path('bul/', views.query, name='spellchecker_query'), + + path('bul/', views.query, name='spellchecker_query'), + path('cevap/', views.answer, name='spellchecker_answer') ] \ No newline at end of file diff --git a/game_project/spellchecker/views.py b/game_project/spellchecker/views.py index a1b66ba..95b10f9 100644 --- a/game_project/spellchecker/views.py +++ b/game_project/spellchecker/views.py @@ -18,28 +18,26 @@ flair.device = torch.device('cpu') classifier = SequenceTagger.load_from_file(url) -def query(request): +def query(request): if request.method == 'POST': form = QueryForm(request.POST) if form.is_valid(): - sentence = form.cleaned_data['sentence'] request.session['sentence'] = sentence - + query = Query(sentence=sentence) if request.user.is_authenticated: - query.user = request.user - + query.user = request.user + query.save() request.session['query_pk'] = query.pk - return redirect('spellchecker_answer') @@ -48,46 +46,40 @@ def query(request): form = QueryForm() - return render(request, 'spellchecker/spellchecker_query.html') -def answer(request): - - sentence = request.session.get('sentence') - - labeled_words, is_error_found = spellchecker(sentence) - - context = { - 'labeled_words': labeled_words, - 'is_error_found': is_error_found, - } +def answer(request): + sentence = request.session.get('sentence') - if request.method == 'POST': - - form = QueryFeedbackForm(request.POST) - - if form.is_valid(): + labeled_words, is_error_found = spellchecker(sentence) - report = form.cleaned_data['report'] + context = { + 'labeled_words': labeled_words, + 'is_error_found': is_error_found, + } - query_pk = request.session.get('query_pk') - query = Query.objects.filter(pk=query_pk)[0] + if request.method == 'POST': - query.sentence = query.sentence + " | REPORT: " + report + form = QueryFeedbackForm(request.POST) - query.save() + if form.is_valid(): + report = form.cleaned_data['report'] - return redirect('spellchecker_query') + query_pk = request.session.get('query_pk') + query = Query.objects.filter(pk=query_pk)[0] - else: + query.sentence = query.sentence + " | REPORT: " + report - form = QueryFeedbackForm() + query.save() + return redirect('spellchecker_query') + else: + form = QueryFeedbackForm() - return render(request, 'spellchecker/spellchecker_answer.html', context) + return render(request, 'spellchecker/spellchecker_answer.html', context) def spellchecker(sentence): @@ -105,21 +97,18 @@ def spellchecker(sentence): print(tagged_string) + is_error_found = False + idx = 0 for token in tokens: - is_error_found = False - - - idx = 0 - for token in tokens: print(str(idx) + ": " + token) - if token == '': + if token == '': - is_error_found = True + is_error_found = True - labels[idx-1] = True + labels[idx - 1] = True # if de/da is seperate, highlight the word before the de/da as well if len(words[idx - 1]) == 2: @@ -140,4 +129,4 @@ def spellchecker(sentence): labeled_words = [{'word': words[i], 'label': labels[i]} for i in range(len(words))] - return labeled_words, is_error_found + return labeled_words, is_error_found From 709ce8de75957735ef4307e54079bcb6267d3739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?onur=20gu=CC=88ngo=CC=88r?= Date: Thu, 30 Jul 2020 17:00:18 +0300 Subject: [PATCH 3/9] removing submodules which were used for bert pretraining once upon a time --- .gitmodules | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .gitmodules diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 1d6fabe..0000000 --- a/.gitmodules +++ /dev/null @@ -1,6 +0,0 @@ -[submodule "bert_pretraining/bert"] - path = bert_pretraining/bert - url = https://github.com/onurgu/bert.git -[submodule "bert_pretraining/bert_google"] - path = bert_pretraining/bert_google - url = https://github.com/haozturk/bert.git From 4f594a779ce4848273db788f089895fac088dc50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?onur=20gu=CC=88ngo=CC=88r?= Date: Thu, 30 Jul 2020 17:07:47 +0300 Subject: [PATCH 4/9] moved bert and flair training dirs into training --- .gitmodules | 4 ++-- .../bert_pretraining}/BERT_PRETRAINING.md | 0 {bert_pretraining => training/bert_pretraining}/bert | 0 .../bert_pretraining}/bert_config_small_turkish.json | 0 {bert_pretraining => training/bert_pretraining}/bert_google | 0 .../bert_pretraining}/correct_punctiation_errors.sh | 0 .../bert_pretraining}/rename_files.py | 0 .../bert_pretraining}/run_create_pretraining_data.sh | 0 .../bert_pretraining}/run_create_pretraining_data_chunks.sh | 0 .../bert_pretraining}/run_extract_features.sh | 0 .../bert_pretraining}/run_pretraining.sh | 0 .../bert_pretraining}/run_sentencepiece_trainer.sh | 0 .../bert_pretraining}/sentence_tokenizer.py | 0 .../bert_pretraining}/sentencepiece_trainer.py | 0 .../bert_pretraining}/vocab_converter.py | 0 .../flair_training}/bak/parameter_optimizer.py | 0 {flair_training => training/flair_training}/bak/train_bert.py | 0 .../flair_training}/bak/train_flair.py | 0 .../flair_training}/bak/train_flair_gpu.py | 0 .../flair_training}/convert_embedding_from_glove_to_gensim.py | 0 .../convert_embedding_from_word2vec_to_gensim.py | 0 {flair_training => training/flair_training}/convert_vocab.py | 0 {flair_training => training/flair_training}/train.py | 0 23 files changed, 2 insertions(+), 2 deletions(-) rename {bert_pretraining => training/bert_pretraining}/BERT_PRETRAINING.md (100%) rename {bert_pretraining => training/bert_pretraining}/bert (100%) rename {bert_pretraining => training/bert_pretraining}/bert_config_small_turkish.json (100%) rename {bert_pretraining => training/bert_pretraining}/bert_google (100%) rename {bert_pretraining => training/bert_pretraining}/correct_punctiation_errors.sh (100%) rename {bert_pretraining => training/bert_pretraining}/rename_files.py (100%) rename {bert_pretraining => training/bert_pretraining}/run_create_pretraining_data.sh (100%) rename {bert_pretraining => training/bert_pretraining}/run_create_pretraining_data_chunks.sh (100%) rename {bert_pretraining => training/bert_pretraining}/run_extract_features.sh (100%) rename {bert_pretraining => training/bert_pretraining}/run_pretraining.sh (100%) rename {bert_pretraining => training/bert_pretraining}/run_sentencepiece_trainer.sh (100%) rename {bert_pretraining => training/bert_pretraining}/sentence_tokenizer.py (100%) rename {bert_pretraining => training/bert_pretraining}/sentencepiece_trainer.py (100%) rename {bert_pretraining => training/bert_pretraining}/vocab_converter.py (100%) rename {flair_training => training/flair_training}/bak/parameter_optimizer.py (100%) rename {flair_training => training/flair_training}/bak/train_bert.py (100%) rename {flair_training => training/flair_training}/bak/train_flair.py (100%) rename {flair_training => training/flair_training}/bak/train_flair_gpu.py (100%) rename {flair_training => training/flair_training}/convert_embedding_from_glove_to_gensim.py (100%) rename {flair_training => training/flair_training}/convert_embedding_from_word2vec_to_gensim.py (100%) rename {flair_training => training/flair_training}/convert_vocab.py (100%) rename {flair_training => training/flair_training}/train.py (100%) diff --git a/.gitmodules b/.gitmodules index 1d6fabe..53c41c9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "bert_pretraining/bert"] - path = bert_pretraining/bert + path = training/bert_pretraining/bert url = https://github.com/onurgu/bert.git [submodule "bert_pretraining/bert_google"] - path = bert_pretraining/bert_google + path = training/bert_pretraining/bert_google url = https://github.com/haozturk/bert.git diff --git a/bert_pretraining/BERT_PRETRAINING.md b/training/bert_pretraining/BERT_PRETRAINING.md similarity index 100% rename from bert_pretraining/BERT_PRETRAINING.md rename to training/bert_pretraining/BERT_PRETRAINING.md diff --git a/bert_pretraining/bert b/training/bert_pretraining/bert similarity index 100% rename from bert_pretraining/bert rename to training/bert_pretraining/bert diff --git a/bert_pretraining/bert_config_small_turkish.json b/training/bert_pretraining/bert_config_small_turkish.json similarity index 100% rename from bert_pretraining/bert_config_small_turkish.json rename to training/bert_pretraining/bert_config_small_turkish.json diff --git a/bert_pretraining/bert_google b/training/bert_pretraining/bert_google similarity index 100% rename from bert_pretraining/bert_google rename to training/bert_pretraining/bert_google diff --git a/bert_pretraining/correct_punctiation_errors.sh b/training/bert_pretraining/correct_punctiation_errors.sh similarity index 100% rename from bert_pretraining/correct_punctiation_errors.sh rename to training/bert_pretraining/correct_punctiation_errors.sh diff --git a/bert_pretraining/rename_files.py b/training/bert_pretraining/rename_files.py similarity index 100% rename from bert_pretraining/rename_files.py rename to training/bert_pretraining/rename_files.py diff --git a/bert_pretraining/run_create_pretraining_data.sh b/training/bert_pretraining/run_create_pretraining_data.sh similarity index 100% rename from bert_pretraining/run_create_pretraining_data.sh rename to training/bert_pretraining/run_create_pretraining_data.sh diff --git a/bert_pretraining/run_create_pretraining_data_chunks.sh b/training/bert_pretraining/run_create_pretraining_data_chunks.sh similarity index 100% rename from bert_pretraining/run_create_pretraining_data_chunks.sh rename to training/bert_pretraining/run_create_pretraining_data_chunks.sh diff --git a/bert_pretraining/run_extract_features.sh b/training/bert_pretraining/run_extract_features.sh similarity index 100% rename from bert_pretraining/run_extract_features.sh rename to training/bert_pretraining/run_extract_features.sh diff --git a/bert_pretraining/run_pretraining.sh b/training/bert_pretraining/run_pretraining.sh similarity index 100% rename from bert_pretraining/run_pretraining.sh rename to training/bert_pretraining/run_pretraining.sh diff --git a/bert_pretraining/run_sentencepiece_trainer.sh b/training/bert_pretraining/run_sentencepiece_trainer.sh similarity index 100% rename from bert_pretraining/run_sentencepiece_trainer.sh rename to training/bert_pretraining/run_sentencepiece_trainer.sh diff --git a/bert_pretraining/sentence_tokenizer.py b/training/bert_pretraining/sentence_tokenizer.py similarity index 100% rename from bert_pretraining/sentence_tokenizer.py rename to training/bert_pretraining/sentence_tokenizer.py diff --git a/bert_pretraining/sentencepiece_trainer.py b/training/bert_pretraining/sentencepiece_trainer.py similarity index 100% rename from bert_pretraining/sentencepiece_trainer.py rename to training/bert_pretraining/sentencepiece_trainer.py diff --git a/bert_pretraining/vocab_converter.py b/training/bert_pretraining/vocab_converter.py similarity index 100% rename from bert_pretraining/vocab_converter.py rename to training/bert_pretraining/vocab_converter.py diff --git a/flair_training/bak/parameter_optimizer.py b/training/flair_training/bak/parameter_optimizer.py similarity index 100% rename from flair_training/bak/parameter_optimizer.py rename to training/flair_training/bak/parameter_optimizer.py diff --git a/flair_training/bak/train_bert.py b/training/flair_training/bak/train_bert.py similarity index 100% rename from flair_training/bak/train_bert.py rename to training/flair_training/bak/train_bert.py diff --git a/flair_training/bak/train_flair.py b/training/flair_training/bak/train_flair.py similarity index 100% rename from flair_training/bak/train_flair.py rename to training/flair_training/bak/train_flair.py diff --git a/flair_training/bak/train_flair_gpu.py b/training/flair_training/bak/train_flair_gpu.py similarity index 100% rename from flair_training/bak/train_flair_gpu.py rename to training/flair_training/bak/train_flair_gpu.py diff --git a/flair_training/convert_embedding_from_glove_to_gensim.py b/training/flair_training/convert_embedding_from_glove_to_gensim.py similarity index 100% rename from flair_training/convert_embedding_from_glove_to_gensim.py rename to training/flair_training/convert_embedding_from_glove_to_gensim.py diff --git a/flair_training/convert_embedding_from_word2vec_to_gensim.py b/training/flair_training/convert_embedding_from_word2vec_to_gensim.py similarity index 100% rename from flair_training/convert_embedding_from_word2vec_to_gensim.py rename to training/flair_training/convert_embedding_from_word2vec_to_gensim.py diff --git a/flair_training/convert_vocab.py b/training/flair_training/convert_vocab.py similarity index 100% rename from flair_training/convert_vocab.py rename to training/flair_training/convert_vocab.py diff --git a/flair_training/train.py b/training/flair_training/train.py similarity index 100% rename from flair_training/train.py rename to training/flair_training/train.py From aac37ef2f7c3cb1528bffdbaf014ced7a21801e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?onur=20gu=CC=88ngo=CC=88r?= Date: Thu, 30 Jul 2020 17:14:16 +0300 Subject: [PATCH 5/9] more relocations --- .../error_analysis}/bertcustom_2020-01-09_02_false_word_freq | 0 .../error_analysis}/bertcustom_2020-01-09_02_true_word_freq | 0 .../error_analysis}/bertcustom_2020-01-09_02_type1_errors | 0 .../error_analysis}/bertcustom_2020-01-09_02_type1_errors_probs | 0 .../error_analysis}/bertcustom_2020-01-09_02_type2_errors | 0 .../error_analysis}/bertcustom_2020-01-09_02_type2_errors_probs | 0 .../error_analysis}/scripts/analyze_colocation.py | 0 .../game_data_analysis}/Game_Data_Analysis.ipynb | 0 {perturbation => analysis/perturbation}/README.md | 0 {perturbation => analysis/perturbation}/perturb.py | 0 {scripts => analysis/scripts}/conll_sample_creator.py | 0 .../convert_bert_tensorflow_checkpoint_to_pytorch_model.sh | 0 {scripts => analysis/scripts}/convert_tf_to_pytorch.sh | 0 {scripts => analysis/scripts}/create-corpus.sh | 0 {scripts => analysis/scripts}/export_data_to_google_sheets.py | 0 {scripts => analysis/scripts}/extract_errors.py | 0 {scripts => analysis/scripts}/morph-analyze.sh | 0 {scripts => analysis/scripts}/sentence_tokenizer.py | 0 {test => analysis/test}/test_baseline_models.py | 0 .../training}/bert_pretraining/BERT_PRETRAINING.md | 0 {training => analysis/training}/bert_pretraining/bert | 0 .../training}/bert_pretraining/bert_config_small_turkish.json | 0 {training => analysis/training}/bert_pretraining/bert_google | 0 .../training}/bert_pretraining/correct_punctiation_errors.sh | 0 {training => analysis/training}/bert_pretraining/rename_files.py | 0 .../training}/bert_pretraining/run_create_pretraining_data.sh | 0 .../bert_pretraining/run_create_pretraining_data_chunks.sh | 0 .../training}/bert_pretraining/run_extract_features.sh | 0 .../training}/bert_pretraining/run_pretraining.sh | 0 .../training}/bert_pretraining/run_sentencepiece_trainer.sh | 0 .../training}/bert_pretraining/sentence_tokenizer.py | 0 .../training}/bert_pretraining/sentencepiece_trainer.py | 0 .../training}/bert_pretraining/vocab_converter.py | 0 .../training}/flair_training/bak/parameter_optimizer.py | 0 {training => analysis/training}/flair_training/bak/train_bert.py | 0 {training => analysis/training}/flair_training/bak/train_flair.py | 0 .../training}/flair_training/bak/train_flair_gpu.py | 0 .../flair_training/convert_embedding_from_glove_to_gensim.py | 0 .../flair_training/convert_embedding_from_word2vec_to_gensim.py | 0 {training => analysis/training}/flair_training/convert_vocab.py | 0 {training => analysis/training}/flair_training/train.py | 0 {xnlp => analysis/xnlp}/__init__.py | 0 {xnlp => analysis/xnlp}/interpret.py | 0 43 files changed, 0 insertions(+), 0 deletions(-) rename {error_analysis => analysis/error_analysis}/bertcustom_2020-01-09_02_false_word_freq (100%) rename {error_analysis => analysis/error_analysis}/bertcustom_2020-01-09_02_true_word_freq (100%) rename {error_analysis => analysis/error_analysis}/bertcustom_2020-01-09_02_type1_errors (100%) rename {error_analysis => analysis/error_analysis}/bertcustom_2020-01-09_02_type1_errors_probs (100%) rename {error_analysis => analysis/error_analysis}/bertcustom_2020-01-09_02_type2_errors (100%) rename {error_analysis => analysis/error_analysis}/bertcustom_2020-01-09_02_type2_errors_probs (100%) rename {error_analysis => analysis/error_analysis}/scripts/analyze_colocation.py (100%) rename {game_data_analysis => analysis/game_data_analysis}/Game_Data_Analysis.ipynb (100%) rename {perturbation => analysis/perturbation}/README.md (100%) rename {perturbation => analysis/perturbation}/perturb.py (100%) rename {scripts => analysis/scripts}/conll_sample_creator.py (100%) rename {scripts => analysis/scripts}/convert_bert_tensorflow_checkpoint_to_pytorch_model.sh (100%) rename {scripts => analysis/scripts}/convert_tf_to_pytorch.sh (100%) rename {scripts => analysis/scripts}/create-corpus.sh (100%) rename {scripts => analysis/scripts}/export_data_to_google_sheets.py (100%) rename {scripts => analysis/scripts}/extract_errors.py (100%) rename {scripts => analysis/scripts}/morph-analyze.sh (100%) rename {scripts => analysis/scripts}/sentence_tokenizer.py (100%) rename {test => analysis/test}/test_baseline_models.py (100%) rename {training => analysis/training}/bert_pretraining/BERT_PRETRAINING.md (100%) rename {training => analysis/training}/bert_pretraining/bert (100%) rename {training => analysis/training}/bert_pretraining/bert_config_small_turkish.json (100%) rename {training => analysis/training}/bert_pretraining/bert_google (100%) rename {training => analysis/training}/bert_pretraining/correct_punctiation_errors.sh (100%) rename {training => analysis/training}/bert_pretraining/rename_files.py (100%) rename {training => analysis/training}/bert_pretraining/run_create_pretraining_data.sh (100%) rename {training => analysis/training}/bert_pretraining/run_create_pretraining_data_chunks.sh (100%) rename {training => analysis/training}/bert_pretraining/run_extract_features.sh (100%) rename {training => analysis/training}/bert_pretraining/run_pretraining.sh (100%) rename {training => analysis/training}/bert_pretraining/run_sentencepiece_trainer.sh (100%) rename {training => analysis/training}/bert_pretraining/sentence_tokenizer.py (100%) rename {training => analysis/training}/bert_pretraining/sentencepiece_trainer.py (100%) rename {training => analysis/training}/bert_pretraining/vocab_converter.py (100%) rename {training => analysis/training}/flair_training/bak/parameter_optimizer.py (100%) rename {training => analysis/training}/flair_training/bak/train_bert.py (100%) rename {training => analysis/training}/flair_training/bak/train_flair.py (100%) rename {training => analysis/training}/flair_training/bak/train_flair_gpu.py (100%) rename {training => analysis/training}/flair_training/convert_embedding_from_glove_to_gensim.py (100%) rename {training => analysis/training}/flair_training/convert_embedding_from_word2vec_to_gensim.py (100%) rename {training => analysis/training}/flair_training/convert_vocab.py (100%) rename {training => analysis/training}/flair_training/train.py (100%) rename {xnlp => analysis/xnlp}/__init__.py (100%) rename {xnlp => analysis/xnlp}/interpret.py (100%) diff --git a/error_analysis/bertcustom_2020-01-09_02_false_word_freq b/analysis/error_analysis/bertcustom_2020-01-09_02_false_word_freq similarity index 100% rename from error_analysis/bertcustom_2020-01-09_02_false_word_freq rename to analysis/error_analysis/bertcustom_2020-01-09_02_false_word_freq diff --git a/error_analysis/bertcustom_2020-01-09_02_true_word_freq b/analysis/error_analysis/bertcustom_2020-01-09_02_true_word_freq similarity index 100% rename from error_analysis/bertcustom_2020-01-09_02_true_word_freq rename to analysis/error_analysis/bertcustom_2020-01-09_02_true_word_freq diff --git a/error_analysis/bertcustom_2020-01-09_02_type1_errors b/analysis/error_analysis/bertcustom_2020-01-09_02_type1_errors similarity index 100% rename from error_analysis/bertcustom_2020-01-09_02_type1_errors rename to analysis/error_analysis/bertcustom_2020-01-09_02_type1_errors diff --git a/error_analysis/bertcustom_2020-01-09_02_type1_errors_probs b/analysis/error_analysis/bertcustom_2020-01-09_02_type1_errors_probs similarity index 100% rename from error_analysis/bertcustom_2020-01-09_02_type1_errors_probs rename to analysis/error_analysis/bertcustom_2020-01-09_02_type1_errors_probs diff --git a/error_analysis/bertcustom_2020-01-09_02_type2_errors b/analysis/error_analysis/bertcustom_2020-01-09_02_type2_errors similarity index 100% rename from error_analysis/bertcustom_2020-01-09_02_type2_errors rename to analysis/error_analysis/bertcustom_2020-01-09_02_type2_errors diff --git a/error_analysis/bertcustom_2020-01-09_02_type2_errors_probs b/analysis/error_analysis/bertcustom_2020-01-09_02_type2_errors_probs similarity index 100% rename from error_analysis/bertcustom_2020-01-09_02_type2_errors_probs rename to analysis/error_analysis/bertcustom_2020-01-09_02_type2_errors_probs diff --git a/error_analysis/scripts/analyze_colocation.py b/analysis/error_analysis/scripts/analyze_colocation.py similarity index 100% rename from error_analysis/scripts/analyze_colocation.py rename to analysis/error_analysis/scripts/analyze_colocation.py diff --git a/game_data_analysis/Game_Data_Analysis.ipynb b/analysis/game_data_analysis/Game_Data_Analysis.ipynb similarity index 100% rename from game_data_analysis/Game_Data_Analysis.ipynb rename to analysis/game_data_analysis/Game_Data_Analysis.ipynb diff --git a/perturbation/README.md b/analysis/perturbation/README.md similarity index 100% rename from perturbation/README.md rename to analysis/perturbation/README.md diff --git a/perturbation/perturb.py b/analysis/perturbation/perturb.py similarity index 100% rename from perturbation/perturb.py rename to analysis/perturbation/perturb.py diff --git a/scripts/conll_sample_creator.py b/analysis/scripts/conll_sample_creator.py similarity index 100% rename from scripts/conll_sample_creator.py rename to analysis/scripts/conll_sample_creator.py diff --git a/scripts/convert_bert_tensorflow_checkpoint_to_pytorch_model.sh b/analysis/scripts/convert_bert_tensorflow_checkpoint_to_pytorch_model.sh similarity index 100% rename from scripts/convert_bert_tensorflow_checkpoint_to_pytorch_model.sh rename to analysis/scripts/convert_bert_tensorflow_checkpoint_to_pytorch_model.sh diff --git a/scripts/convert_tf_to_pytorch.sh b/analysis/scripts/convert_tf_to_pytorch.sh similarity index 100% rename from scripts/convert_tf_to_pytorch.sh rename to analysis/scripts/convert_tf_to_pytorch.sh diff --git a/scripts/create-corpus.sh b/analysis/scripts/create-corpus.sh similarity index 100% rename from scripts/create-corpus.sh rename to analysis/scripts/create-corpus.sh diff --git a/scripts/export_data_to_google_sheets.py b/analysis/scripts/export_data_to_google_sheets.py similarity index 100% rename from scripts/export_data_to_google_sheets.py rename to analysis/scripts/export_data_to_google_sheets.py diff --git a/scripts/extract_errors.py b/analysis/scripts/extract_errors.py similarity index 100% rename from scripts/extract_errors.py rename to analysis/scripts/extract_errors.py diff --git a/scripts/morph-analyze.sh b/analysis/scripts/morph-analyze.sh similarity index 100% rename from scripts/morph-analyze.sh rename to analysis/scripts/morph-analyze.sh diff --git a/scripts/sentence_tokenizer.py b/analysis/scripts/sentence_tokenizer.py similarity index 100% rename from scripts/sentence_tokenizer.py rename to analysis/scripts/sentence_tokenizer.py diff --git a/test/test_baseline_models.py b/analysis/test/test_baseline_models.py similarity index 100% rename from test/test_baseline_models.py rename to analysis/test/test_baseline_models.py diff --git a/training/bert_pretraining/BERT_PRETRAINING.md b/analysis/training/bert_pretraining/BERT_PRETRAINING.md similarity index 100% rename from training/bert_pretraining/BERT_PRETRAINING.md rename to analysis/training/bert_pretraining/BERT_PRETRAINING.md diff --git a/training/bert_pretraining/bert b/analysis/training/bert_pretraining/bert similarity index 100% rename from training/bert_pretraining/bert rename to analysis/training/bert_pretraining/bert diff --git a/training/bert_pretraining/bert_config_small_turkish.json b/analysis/training/bert_pretraining/bert_config_small_turkish.json similarity index 100% rename from training/bert_pretraining/bert_config_small_turkish.json rename to analysis/training/bert_pretraining/bert_config_small_turkish.json diff --git a/training/bert_pretraining/bert_google b/analysis/training/bert_pretraining/bert_google similarity index 100% rename from training/bert_pretraining/bert_google rename to analysis/training/bert_pretraining/bert_google diff --git a/training/bert_pretraining/correct_punctiation_errors.sh b/analysis/training/bert_pretraining/correct_punctiation_errors.sh similarity index 100% rename from training/bert_pretraining/correct_punctiation_errors.sh rename to analysis/training/bert_pretraining/correct_punctiation_errors.sh diff --git a/training/bert_pretraining/rename_files.py b/analysis/training/bert_pretraining/rename_files.py similarity index 100% rename from training/bert_pretraining/rename_files.py rename to analysis/training/bert_pretraining/rename_files.py diff --git a/training/bert_pretraining/run_create_pretraining_data.sh b/analysis/training/bert_pretraining/run_create_pretraining_data.sh similarity index 100% rename from training/bert_pretraining/run_create_pretraining_data.sh rename to analysis/training/bert_pretraining/run_create_pretraining_data.sh diff --git a/training/bert_pretraining/run_create_pretraining_data_chunks.sh b/analysis/training/bert_pretraining/run_create_pretraining_data_chunks.sh similarity index 100% rename from training/bert_pretraining/run_create_pretraining_data_chunks.sh rename to analysis/training/bert_pretraining/run_create_pretraining_data_chunks.sh diff --git a/training/bert_pretraining/run_extract_features.sh b/analysis/training/bert_pretraining/run_extract_features.sh similarity index 100% rename from training/bert_pretraining/run_extract_features.sh rename to analysis/training/bert_pretraining/run_extract_features.sh diff --git a/training/bert_pretraining/run_pretraining.sh b/analysis/training/bert_pretraining/run_pretraining.sh similarity index 100% rename from training/bert_pretraining/run_pretraining.sh rename to analysis/training/bert_pretraining/run_pretraining.sh diff --git a/training/bert_pretraining/run_sentencepiece_trainer.sh b/analysis/training/bert_pretraining/run_sentencepiece_trainer.sh similarity index 100% rename from training/bert_pretraining/run_sentencepiece_trainer.sh rename to analysis/training/bert_pretraining/run_sentencepiece_trainer.sh diff --git a/training/bert_pretraining/sentence_tokenizer.py b/analysis/training/bert_pretraining/sentence_tokenizer.py similarity index 100% rename from training/bert_pretraining/sentence_tokenizer.py rename to analysis/training/bert_pretraining/sentence_tokenizer.py diff --git a/training/bert_pretraining/sentencepiece_trainer.py b/analysis/training/bert_pretraining/sentencepiece_trainer.py similarity index 100% rename from training/bert_pretraining/sentencepiece_trainer.py rename to analysis/training/bert_pretraining/sentencepiece_trainer.py diff --git a/training/bert_pretraining/vocab_converter.py b/analysis/training/bert_pretraining/vocab_converter.py similarity index 100% rename from training/bert_pretraining/vocab_converter.py rename to analysis/training/bert_pretraining/vocab_converter.py diff --git a/training/flair_training/bak/parameter_optimizer.py b/analysis/training/flair_training/bak/parameter_optimizer.py similarity index 100% rename from training/flair_training/bak/parameter_optimizer.py rename to analysis/training/flair_training/bak/parameter_optimizer.py diff --git a/training/flair_training/bak/train_bert.py b/analysis/training/flair_training/bak/train_bert.py similarity index 100% rename from training/flair_training/bak/train_bert.py rename to analysis/training/flair_training/bak/train_bert.py diff --git a/training/flair_training/bak/train_flair.py b/analysis/training/flair_training/bak/train_flair.py similarity index 100% rename from training/flair_training/bak/train_flair.py rename to analysis/training/flair_training/bak/train_flair.py diff --git a/training/flair_training/bak/train_flair_gpu.py b/analysis/training/flair_training/bak/train_flair_gpu.py similarity index 100% rename from training/flair_training/bak/train_flair_gpu.py rename to analysis/training/flair_training/bak/train_flair_gpu.py diff --git a/training/flair_training/convert_embedding_from_glove_to_gensim.py b/analysis/training/flair_training/convert_embedding_from_glove_to_gensim.py similarity index 100% rename from training/flair_training/convert_embedding_from_glove_to_gensim.py rename to analysis/training/flair_training/convert_embedding_from_glove_to_gensim.py diff --git a/training/flair_training/convert_embedding_from_word2vec_to_gensim.py b/analysis/training/flair_training/convert_embedding_from_word2vec_to_gensim.py similarity index 100% rename from training/flair_training/convert_embedding_from_word2vec_to_gensim.py rename to analysis/training/flair_training/convert_embedding_from_word2vec_to_gensim.py diff --git a/training/flair_training/convert_vocab.py b/analysis/training/flair_training/convert_vocab.py similarity index 100% rename from training/flair_training/convert_vocab.py rename to analysis/training/flair_training/convert_vocab.py diff --git a/training/flair_training/train.py b/analysis/training/flair_training/train.py similarity index 100% rename from training/flair_training/train.py rename to analysis/training/flair_training/train.py diff --git a/xnlp/__init__.py b/analysis/xnlp/__init__.py similarity index 100% rename from xnlp/__init__.py rename to analysis/xnlp/__init__.py diff --git a/xnlp/interpret.py b/analysis/xnlp/interpret.py similarity index 100% rename from xnlp/interpret.py rename to analysis/xnlp/interpret.py From e4007b67bd08e5fa9483c25264a39a9607745441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?onur=20gu=CC=88ngo=CC=88r?= Date: Thu, 30 Jul 2020 18:22:56 +0300 Subject: [PATCH 6/9] =?UTF-8?q?F=C4=B1xed=20an=20issue=20in:=20Reverted=20?= =?UTF-8?q?to=20the=20version=20in=20master=20branch,=20will=20restart=20w?= =?UTF-8?q?ork=20from=20here?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../spellchecker/spellchecker_answer.html | 2 +- .../spellchecker/spellchecker_query.html | 74 +++++-------------- 2 files changed, 21 insertions(+), 55 deletions(-) diff --git a/game_project/spellchecker/templates/spellchecker/spellchecker_answer.html b/game_project/spellchecker/templates/spellchecker/spellchecker_answer.html index 7de343d..7c7a250 100644 --- a/game_project/spellchecker/templates/spellchecker/spellchecker_answer.html +++ b/game_project/spellchecker/templates/spellchecker/spellchecker_answer.html @@ -96,4 +96,4 @@

Hata Bildir

-{% endblock content %} \ No newline at end of file +{% endblock content %} diff --git a/game_project/spellchecker/templates/spellchecker/spellchecker_query.html b/game_project/spellchecker/templates/spellchecker/spellchecker_query.html index 11be655..97e884f 100644 --- a/game_project/spellchecker/templates/spellchecker/spellchecker_query.html +++ b/game_project/spellchecker/templates/spellchecker/spellchecker_query.html @@ -1,7 +1,7 @@ {% extends 'game/base.html' %} {% load crispy_forms_tags %} {% block content %} - +
@@ -10,50 +10,19 @@

-de/-da Yazım Hatası Bulucu



- {% if labeled_words %} -
- - {% for entry in labeled_words %} - - {% if entry.label == True %} - - {{entry.word}} - - {% else %} - - {{entry.word}} - - {% endif %} - - {% endfor %} +
+ {% csrf_token %} -
+ -
- -
- + - {% else %} +
+
- - {% csrf_token %} +

+ - - - - -
-
- -

- - - {% endif %}
@@ -66,22 +35,19 @@

-de/-da Yazım Hatası Bulucu

- {% if not labeled_words %} -
- - - - -
- {% endif %} + + + + +
-

diff --git a/game_project/game/templates/game/base.html b/game_project/game/templates/game/base.html index 6c982a5..8bd7d69 100644 --- a/game_project/game/templates/game/base.html +++ b/game_project/game/templates/game/base.html @@ -27,6 +27,14 @@ gtag('js', new Date()); gtag('config', 'UA-170798910-2'); + @@ -40,7 +48,7 @@ - -

diff --git a/game_project/game/templates/game/welcome.html b/game_project/game/templates/game/welcome.html index 83215b9..ba1298b 100644 --- a/game_project/game/templates/game/welcome.html +++ b/game_project/game/templates/game/welcome.html @@ -15,7 +15,7 @@

Hoş Geldiniz

Araştırma ekibimiz tarafından Boğaziçi Üniversitesi Bilgisayar Mühendisliği çatısı altında yapay zeka yöntemleri kullanarak "de/da" yazım hatalarını tespit eden bir bilgisayar uygulaması geliştirilmişti. Şimdi ise "de/da" yazım hatalarının doğasını ve geliştirdiğimiz uygulamanın kararlarını daha iyi anlamak amacıyla bir deney yapıyoruz. - İsterseniz deneyimize katılmadan önce "-de/-da Yazım Hatası Bulucu" sayfasında geliştirdiğimiz uygulamayı deneyebilirsiniz. + İsterseniz deneyimize katılmadan önce "-de/-da Yazım Hatası Bulucu" sayfasında geliştirdiğimiz uygulamayı deneyebilirsiniz. Eğer daha önce kayıt olmadıysanız kayıt olarak deneye başlayabilir, eğer kayıt olmuşsanız giriş yaparak deneye kaldığınız yerden devam edebilirsiniz. diff --git a/game_project/spellchecker/forms.py b/game_project/spellchecker/forms.py index 9461bca..3f561af 100644 --- a/game_project/spellchecker/forms.py +++ b/game_project/spellchecker/forms.py @@ -3,8 +3,9 @@ class QueryForm(forms.Form): sentence = forms.CharField(max_length=500) + is_error_found = forms.BooleanField(required=False) + tagged_string = forms.TextInput() class QueryFeedbackForm(forms.Form): - #isHappy = forms.BooleanField(required = False) report = forms.CharField(max_length=1000) \ No newline at end of file diff --git a/game_project/spellchecker/migrations/0005_auto_20200730_1913.py b/game_project/spellchecker/migrations/0005_auto_20200730_1913.py new file mode 100644 index 0000000..270f03d --- /dev/null +++ b/game_project/spellchecker/migrations/0005_auto_20200730_1913.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.5 on 2020-07-30 19:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('spellchecker', '0004_auto_20200531_0903'), + ] + + operations = [ + migrations.AddField( + model_name='query', + name='tagged_sentence', + field=models.TextField(default=''), + ), + migrations.AlterField( + model_name='query', + name='sentence', + field=models.TextField(default=''), + ), + ] diff --git a/game_project/spellchecker/migrations/0006_feedback.py b/game_project/spellchecker/migrations/0006_feedback.py new file mode 100644 index 0000000..f5bcea6 --- /dev/null +++ b/game_project/spellchecker/migrations/0006_feedback.py @@ -0,0 +1,22 @@ +# Generated by Django 3.0.5 on 2020-07-31 13:30 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('spellchecker', '0005_auto_20200730_1913'), + ] + + operations = [ + migrations.CreateModel( + name='Feedback', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('report', models.TextField(null=True)), + ('query', models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, to='spellchecker.Query')), + ], + ), + ] diff --git a/game_project/spellchecker/models.py b/game_project/spellchecker/models.py index 9785648..2a78797 100644 --- a/game_project/spellchecker/models.py +++ b/game_project/spellchecker/models.py @@ -1,8 +1,15 @@ from django.db import models from django.contrib.auth.models import User + class Query(models.Model): - sentence = models.TextField() - isHappy = models.BooleanField( blank=True, null=True) - user = models.ForeignKey(User, blank=True, null=True, on_delete=models.CASCADE) \ No newline at end of file + sentence = models.TextField(default="") + user = models.ForeignKey(User, blank=True, null=True, on_delete=models.CASCADE) + tagged_sentence = models.TextField(default="") + + +class Feedback(models.Model): + + query = models.ForeignKey(Query, on_delete=models.DO_NOTHING) + report = models.TextField(null=True) diff --git a/game_project/spellchecker/templates/spellchecker/spellchecker_answer.html b/game_project/spellchecker/templates/spellchecker/spellchecker_answer.html deleted file mode 100644 index 7c7a250..0000000 --- a/game_project/spellchecker/templates/spellchecker/spellchecker_answer.html +++ /dev/null @@ -1,99 +0,0 @@ -{% extends 'game/base.html' %} -{% load static %} -{% block content %} - - -
- -

-de/-da Yazım Hatası Bulucu

- -
-
- -
- - {% for entry in labeled_words %} - - {% if entry.label == True %} - - {{entry.word}} - - {% else %} - - {{entry.word}} - - {% endif %} - - {% endfor %} - -
- -
- -
- - {% if is_error_found %} - - {% else %} - -

Hata bulunamadı.

- - - {% endif %} - -
- -
- - - -
-
- -
- - -
- - - -
- - - -{% endblock content %} diff --git a/game_project/spellchecker/templates/spellchecker/spellchecker_query.html b/game_project/spellchecker/templates/spellchecker/spellchecker_query.html index 97e884f..2786265 100644 --- a/game_project/spellchecker/templates/spellchecker/spellchecker_query.html +++ b/game_project/spellchecker/templates/spellchecker/spellchecker_query.html @@ -10,18 +10,90 @@

-de/-da Yazım Hatası Bulucu



-
- {% csrf_token %} + {% if labeled_words %} +
- + {% for entry in labeled_words %} - + {% if entry.label == True %} -
-
+ {{entry.word}} -

- + {% else %} + + {{entry.word}} + + {% endif %} + + {% endfor %} + +
+ +
+ +
+ + {% if is_error_found %} + + {% else %} + +

Hata bulunamadı.

+ + + {% endif %} + +
+ +
+ + + +
+
+ +
+ + +
+ + + + + {% else %} + +
+ {% csrf_token %} + + + + + +
+
+ +

+
+ + {% endif %} diff --git a/game_project/spellchecker/urls.py b/game_project/spellchecker/urls.py index 08c8290..6ec52d4 100644 --- a/game_project/spellchecker/urls.py +++ b/game_project/spellchecker/urls.py @@ -2,9 +2,10 @@ from . import views +app_name = "spellchecker" urlpatterns = [ - path('bul/', views.query, name='spellchecker_query'), - path('cevap/', views.answer, name='spellchecker_answer') + path('bul/', views.query, name='query'), + path('geribildirim/', views.feedback, name='feedback') ] \ No newline at end of file diff --git a/game_project/spellchecker/views.py b/game_project/spellchecker/views.py index 95b10f9..82b1cca 100644 --- a/game_project/spellchecker/views.py +++ b/game_project/spellchecker/views.py @@ -1,17 +1,21 @@ +from django.http import HttpResponseRedirect, HttpResponseNotAllowed from django.shortcuts import render, redirect +from django.urls import reverse + from .forms import QueryForm, QueryFeedbackForm -from .models import Query import flair, torch from flair.models import SequenceTagger from flair.data import Sentence from django.contrib.staticfiles import finders -from .models import Query +from .models import Query, Feedback from game.forms import ReportForm from game.models import Report +from users.models import User + from game.views import str2bool url = finders.find('best-model.pt') @@ -20,44 +24,43 @@ def query(request): - if request.method == 'POST': - + if request.method == 'GET': + form = QueryForm() + elif request.method == 'POST': form = QueryForm(request.POST) if form.is_valid(): sentence = form.cleaned_data['sentence'] - request.session['sentence'] = sentence - - query = Query(sentence=sentence) + labeled_words, is_error_found, tagged_sentence = invoke_spellchecker_engine(sentence) + query = Query(sentence=sentence, + tagged_sentence=tagged_sentence) if request.user.is_authenticated: query.user = request.user - + else: + query.user = None query.save() request.session['query_pk'] = query.pk - return redirect('spellchecker_answer') - + context = { + 'labeled_words': labeled_words, + 'is_error_found': is_error_found, + 'form': form + } + # return redirect('spellchecker_answer') + return render(request, 'spellchecker/spellchecker_query.html', context) else: - form = QueryForm() - return render(request, 'spellchecker/spellchecker_query.html') - - -def answer(request): - sentence = request.session.get('sentence') + context = {'form': form} + return render(request, 'spellchecker/spellchecker_query.html', context) - labeled_words, is_error_found = spellchecker(sentence) - context = { - 'labeled_words': labeled_words, - 'is_error_found': is_error_found, - } +def feedback(request): if request.method == 'POST': @@ -65,40 +68,36 @@ def answer(request): if form.is_valid(): report = form.cleaned_data['report'] - query_pk = request.session.get('query_pk') - query = Query.objects.filter(pk=query_pk)[0] - - query.sentence = query.sentence + " | REPORT: " + report - - query.save() - - return redirect('spellchecker_query') + query = Query.objects.get(pk=query_pk) + feedback = Feedback(report=report, + query=query) + feedback.save() + return HttpResponseRedirect(reverse('spellchecker:query')) + else: + return HttpResponseRedirect(reverse('welcome')) else: + return HttpResponseNotAllowed(permitted_methods=["POST"]) - form = QueryFeedbackForm() - - return render(request, 'spellchecker/spellchecker_answer.html', context) - -def spellchecker(sentence): - sentence = Sentence(sentence) +def invoke_spellchecker_engine(sentence_str): + sentence = Sentence(sentence_str) classifier.predict(sentence) tagged_string = sentence.to_tagged_string() - labeled_words = {} + is_error_found, labeled_words = parse_tagged_string(tagged_string) - words, labels = [], [] + return labeled_words, is_error_found, tagged_string - tokens = tagged_string.split() +def parse_tagged_string(tagged_string): + words, labels = [], [] + tokens = tagged_string.split() print(tagged_string) - is_error_found = False - idx = 0 for token in tokens: @@ -126,7 +125,5 @@ def spellchecker(sentence): labels.append(False) idx += 1 - labeled_words = [{'word': words[i], 'label': labels[i]} for i in range(len(words))] - - return labeled_words, is_error_found + return is_error_found, labeled_words From a56de73582f4c443a03231f9fe13ab14cdae251a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?onur=20gu=CC=88ngo=CC=88r?= Date: Fri, 31 Jul 2020 17:52:14 +0300 Subject: [PATCH 8/9] - changed tab to 4 spaces --- game_project/users/models.py | 46 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/game_project/users/models.py b/game_project/users/models.py index 210b6e8..d9667c3 100644 --- a/game_project/users/models.py +++ b/game_project/users/models.py @@ -3,37 +3,37 @@ from PIL import Image class Profile(models.Model): - user = models.OneToOneField(User, on_delete=models.CASCADE) - image = models.ImageField(default='default.jpg', upload_to='profile_pics') - #last_seen_sentence_idx = models.IntegerField(default=1) - last_seen_sentence_idx = models.IntegerField(default=0) - correct_answer_count = models.IntegerField(default=0) + user = models.OneToOneField(User, on_delete=models.CASCADE) + image = models.ImageField(default='default.jpg', upload_to='profile_pics') + #last_seen_sentence_idx = models.IntegerField(default=1) + last_seen_sentence_idx = models.IntegerField(default=0) + correct_answer_count = models.IntegerField(default=0) - #onam = models.BooleanField(default=False) - #isOnamSubmitted = models.BooleanField(default=False) + #onam = models.BooleanField(default=False) + #isOnamSubmitted = models.BooleanField(default=False) - #last_seen_prof_idx = models.IntegerField(default=1) - #prof_score = models.IntegerField(default=0) - #is_prof_done = models.BooleanField(default=False) + #last_seen_prof_idx = models.IntegerField(default=1) + #prof_score = models.IntegerField(default=0) + #is_prof_done = models.BooleanField(default=False) - last_seen_warmup_idx = models.IntegerField(default=1) - is_warmup_done = models.BooleanField(default=False) + last_seen_warmup_idx = models.IntegerField(default=1) + is_warmup_done = models.BooleanField(default=False) - tos = models.BooleanField(default=False) # terms of service (onam formu) + tos = models.BooleanField(default=False) # terms of service (onam formu) - def __str__(self): - return f'{self.user.username} Profile' + def __str__(self): + return f'{self.user.username} Profile' - - def save(self, *args, **kwargs): - super().save(*args, **kwargs) - img = Image.open(self.image.path) + def save(self, *args, **kwargs): + super().save(*args, **kwargs) - if img.height > 300 or img.width > 300: - output_size = (300,300) - img.thumbnail(output_size) - img.save(self.image.path) + img = Image.open(self.image.path) + + if img.height > 300 or img.width > 300: + output_size = (300,300) + img.thumbnail(output_size) + img.save(self.image.path) \ No newline at end of file From cd8085228abb30726e6edf6d56e3c83a260d31c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?onur=20gu=CC=88ngo=CC=88r?= Date: Fri, 31 Jul 2020 21:17:52 +0300 Subject: [PATCH 9/9] - a GET request to 'query' view with 'sentence' param invokes the spelling query engine - each query has a UUID named 'random_id', thus the results can be shared as a URL - removed isHappy field from Query I realized that this second functionality removes the need for the first functionality, not sure though. These new changes allow us to make queries in the background and use the 'query_detail' URLs to show them to other people. using clipboard.js for this. - added a custom template tag named 'domain_name' to get the domain name of the deployment. not sure this is the best way though. --- game_project/game/templates/game/base.html | 7 +- game_project/game_project/settings.py | 4 +- .../migrations/0007_auto_20200731_1638.py | 23 +++++ game_project/spellchecker/models.py | 4 + .../spellchecker/spellchecker_query.html | 10 +++ .../spellchecker/templatetags/__init__.py | 0 .../templatetags/global_information_tags.py | 9 ++ game_project/spellchecker/urls.py | 3 +- game_project/spellchecker/views.py | 84 +++++++++++-------- 9 files changed, 107 insertions(+), 37 deletions(-) create mode 100644 game_project/spellchecker/migrations/0007_auto_20200731_1638.py create mode 100644 game_project/spellchecker/templatetags/__init__.py create mode 100644 game_project/spellchecker/templatetags/global_information_tags.py diff --git a/game_project/game/templates/game/base.html b/game_project/game/templates/game/base.html index 8bd7d69..9226480 100644 --- a/game_project/game/templates/game/base.html +++ b/game_project/game/templates/game/base.html @@ -35,6 +35,8 @@ } + + @@ -97,6 +99,9 @@ - + + \ No newline at end of file diff --git a/game_project/game_project/settings.py b/game_project/game_project/settings.py index 3fd5373..3657346 100644 --- a/game_project/game_project/settings.py +++ b/game_project/game_project/settings.py @@ -151,4 +151,6 @@ MESSAGE_TAGS = { messages.ERROR: 'danger', -} \ No newline at end of file +} + +DOMAIN_NAME = 'dedatakintisi.derlem.com' \ No newline at end of file diff --git a/game_project/spellchecker/migrations/0007_auto_20200731_1638.py b/game_project/spellchecker/migrations/0007_auto_20200731_1638.py new file mode 100644 index 0000000..73d1977 --- /dev/null +++ b/game_project/spellchecker/migrations/0007_auto_20200731_1638.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.5 on 2020-07-31 16:38 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('spellchecker', '0006_feedback'), + ] + + operations = [ + migrations.RemoveField( + model_name='query', + name='isHappy', + ), + migrations.AddField( + model_name='query', + name='random_id', + field=models.UUIDField(db_index=True, default=uuid.uuid4, editable=False), + ), + ] diff --git a/game_project/spellchecker/models.py b/game_project/spellchecker/models.py index 2a78797..f415ce4 100644 --- a/game_project/spellchecker/models.py +++ b/game_project/spellchecker/models.py @@ -1,3 +1,5 @@ +import uuid + from django.db import models from django.contrib.auth.models import User @@ -8,6 +10,8 @@ class Query(models.Model): user = models.ForeignKey(User, blank=True, null=True, on_delete=models.CASCADE) tagged_sentence = models.TextField(default="") + random_id = models.UUIDField(db_index=True, default=uuid.uuid4, editable=False) + class Feedback(models.Model): diff --git a/game_project/spellchecker/templates/spellchecker/spellchecker_query.html b/game_project/spellchecker/templates/spellchecker/spellchecker_query.html index 2786265..711a527 100644 --- a/game_project/spellchecker/templates/spellchecker/spellchecker_query.html +++ b/game_project/spellchecker/templates/spellchecker/spellchecker_query.html @@ -1,5 +1,7 @@ {% extends 'game/base.html' %} {% load crispy_forms_tags %} +{% load global_information_tags %} + {% block content %} @@ -44,6 +46,14 @@

-de/-da Yazım Hatası Bulucu

+
+ + + + +
diff --git a/game_project/spellchecker/templatetags/__init__.py b/game_project/spellchecker/templatetags/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/game_project/spellchecker/templatetags/global_information_tags.py b/game_project/spellchecker/templatetags/global_information_tags.py new file mode 100644 index 0000000..285a5dd --- /dev/null +++ b/game_project/spellchecker/templatetags/global_information_tags.py @@ -0,0 +1,9 @@ +from django import template + +register = template.Library() + +from game_project.settings import DOMAIN_NAME + +@register.simple_tag +def domain_name(): + return DOMAIN_NAME \ No newline at end of file diff --git a/game_project/spellchecker/urls.py b/game_project/spellchecker/urls.py index 6ec52d4..6755235 100644 --- a/game_project/spellchecker/urls.py +++ b/game_project/spellchecker/urls.py @@ -6,6 +6,7 @@ urlpatterns = [ path('bul/', views.query, name='query'), - path('geribildirim/', views.feedback, name='feedback') + path('geribildirim/', views.feedback, name='feedback'), + path('bul/', views.query_detail, name='query_detail') ] \ No newline at end of file diff --git a/game_project/spellchecker/views.py b/game_project/spellchecker/views.py index 82b1cca..a782281 100644 --- a/game_project/spellchecker/views.py +++ b/game_project/spellchecker/views.py @@ -1,4 +1,4 @@ -from django.http import HttpResponseRedirect, HttpResponseNotAllowed +from django.http import HttpResponseRedirect, HttpResponseNotAllowed, HttpResponseNotFound from django.shortcuts import render, redirect from django.urls import reverse @@ -11,53 +11,69 @@ from .models import Query, Feedback -from game.forms import ReportForm -from game.models import Report - -from users.models import User - -from game.views import str2bool - url = finders.find('best-model.pt') flair.device = torch.device('cpu') classifier = SequenceTagger.load_from_file(url) +def query_detail(request, random_id): + + query = Query.objects.get(random_id=random_id) + if query: + is_error_found, labeled_words = parse_tagged_string(query.tagged_sentence) + context = { + 'labeled_words': labeled_words, + 'is_error_found': is_error_found, + 'query_random_id': random_id + } + return render(request, 'spellchecker/spellchecker_query.html', context) + else: + return HttpResponseNotFound() + + def query(request): + form = QueryForm() if request.method == 'GET': - form = QueryForm() + if 'sentence' in request.GET: + sentence = request.GET['sentence'] + else: + sentence = None elif request.method == 'POST': form = QueryForm(request.POST) - if form.is_valid(): - sentence = form.cleaned_data['sentence'] + else: + sentence = None + else: + return HttpResponseNotAllowed(permitted_methods=["GET", "POST"]) - labeled_words, is_error_found, tagged_sentence = invoke_spellchecker_engine(sentence) - - query = Query(sentence=sentence, - tagged_sentence=tagged_sentence) - if request.user.is_authenticated: - query.user = request.user - else: - query.user = None - query.save() - - request.session['query_pk'] = query.pk - - context = { - 'labeled_words': labeled_words, - 'is_error_found': is_error_found, - 'form': form - } + if sentence: + labeled_words, is_error_found, tagged_sentence = invoke_spellchecker_engine(sentence) - # return redirect('spellchecker_answer') - return render(request, 'spellchecker/spellchecker_query.html', context) + query = Query(sentence=sentence, + tagged_sentence=tagged_sentence) + if request.user.is_authenticated: + query.user = request.user + else: + query.user = None + query.save() + + query_random_id = str(query.random_id) + request.session['query_random_id'] = query_random_id + request.session['query_pk'] = query.pk + + context = { + 'labeled_words': labeled_words, + 'is_error_found': is_error_found, + 'query_random_id': query_random_id, + 'form': form + } + + # return redirect('spellchecker_answer') + return render(request, 'spellchecker/spellchecker_query.html', context) else: - form = QueryForm() - - context = {'form': form} - return render(request, 'spellchecker/spellchecker_query.html', context) + context = {'form': form} + return render(request, 'spellchecker/spellchecker_query.html', context) def feedback(request):