From cb73396ab08dbb6107d9e888e80030215929fa74 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sat, 17 Aug 2019 06:02:40 -0400 Subject: [PATCH 001/750] Tracking most keystrokes and saves --- extension/background.js | 38 +++++++++++++++ extension/icons/lousy-fountain-pen-48.png | Bin 0 -> 2223 bytes extension/icons/lousy-fountain-pen-48.xcf | Bin 0 -> 5558 bytes extension/manifest.json | 33 +++++++++++++ extension/popup/settings.html | 6 +++ extension/writing.js | 56 ++++++++++++++++++++++ 6 files changed, 133 insertions(+) create mode 100644 extension/background.js create mode 100644 extension/icons/lousy-fountain-pen-48.png create mode 100644 extension/icons/lousy-fountain-pen-48.xcf create mode 100644 extension/manifest.json create mode 100644 extension/popup/settings.html create mode 100644 extension/writing.js diff --git a/extension/background.js b/extension/background.js new file mode 100644 index 000000000..0760f4247 --- /dev/null +++ b/extension/background.js @@ -0,0 +1,38 @@ +var WRITINGJS_SERVER = "https://test.mitros.org/webapi/"; + +function writingjs_ajax(data) { + httpRequest = new XMLHttpRequest(); + //httpRequest.withCredentials = true; + httpRequest.open("POST", WRITINGJS_SERVER); + httpRequest.send(JSON.stringify(data)); +} + +function writing_eventlistener(event) { + var event_data = {}; + event_data["event_type"] = "keypress"; + properties = ['altKey', 'charCode', 'code', 'ctrlKey', 'isComposing', 'key', 'keyCode', 'location', 'metaKey', 'repeat', 'shiftKey', 'which', 'isTrusted', 'timeStemp', 'type']; + for (var property in properties) { + event_data[properties[property]] = event[properties[property]]; + } + event_data['date'] = new Date().toLocaleString('en-US') + console.log(JSON.stringify(event_data)); + writingjs_ajax(event_data); +} + +chrome.webRequest.onBeforeRequest.addListener( + function(request) { + writingjs_ajax({ + 'event_type':'request', + 'url': request.url, + 'from_data': request.requestBody.formData + }); + }, + { urls: ["*://docs.google.com/*"] }, + ['requestBody'] +) + +/*chrome.tabs.executeScript({ + code: 'console.log("addd")' +});*/ + +writingjs_ajax({"Loaded": true}); diff --git a/extension/icons/lousy-fountain-pen-48.png b/extension/icons/lousy-fountain-pen-48.png new file mode 100644 index 0000000000000000000000000000000000000000..223edca88e689dd529fd540c720cc45f589403df GIT binary patch literal 2223 zcmV;g2vGNlP)aTpLcZvH?ngwqR?;I<}qB8PGFt z#ZiyE3Gc((P;bDV)XsDqJB}V$ORbc)ga9FILJ~suoB!bTTVEzn3jO@o5tD@VKI?g( zcehBABng5*2o;OP7cXACdiAPQDiH(`1c7bax~_*|$S{oSx|U^yVMq`JK@b#0EiEl& zG8vX-iJ1%j5{4oCpePC~5(Ht}_Tk~-+qZ8I4h{^%2*Z#hNeaGWS$2DS`^l3hTU%Sy zKa1nIbUK|%rD&Qa2m)`k&>~5aBuUs1Ar#$&W!XD-?rd*wr_*WtoPw`cS654=QmIs0 zT3W&zQ540pETF+hj9FR;+2oT#q440rgO!yPgwP*A1VJziv$eJL^y$-k_wL~r_+$Vm z2!be}sc5iw=0};yWF9_zc=zsInx;vT48xEj2qKkA?drQH3Nr|T7)dcNkpHZO6h={0p-|Y| z+{99$D2hs_)45!ZWm(_%V_E?lQD5M z&QZhB-|ybN%jfe9!ytr`$z-`)o*zmWhOXblQ`dDp&$|(p@B8ENxY=wr8VyK{1VKeu3MJndERV)*|y#5^-fPu&(F`t z<1tvI!!R@qLlnh&y}r4*na}5!mzNoap(qMibzK*`3uAwD2g$XfF1x8WeC>J&d$_~?0KGT+qP{(*iHpx7)GU1IXXJ} z{{6e6DA*@|XwY@N*=(Mjon2mD4h92IQV;~#U(qZ8{vt{8`Sa%-$Hk28?(RY&bsPsm zWD%O6v$M0ew}(HbX&QS!YUt|f>eHuBdwYAofBznj$5F|m7;us#lgZ@T z+FCZ7B?tnv8D)rJm{cm2NF>06<#PGWn>WMZuuv%M@9)2S`7%lgLWpJAXp#3kui0!K z9v*)D`0@DocsLx!G9MjPnXc2xg1f;=)cXHrw!P~Z1W)08BsR;x*pWSS;O zWNL850u7{%pF{BxMX_3~@;tAq>g*avT{pORo;Mne+U<6=T9qWpb=_FcdR+$yfMQKg zyWOtW>!K(|XQ_Ds;geaG)$jM4&1SdTg#m4{RhpWOa3n&YjmP6gqj7O@p=lbX=em8U z8Pgr4%5JwS2!gKbj^kWcETe!x+Bh}%zCRj`E-x?J?e=&)zPZbu-O-t*+3WQLL1?$z zaK0N!g2@RFPY@rSPDfFcXo^}G5GF~}v`(kP^Sms}hG9S=p0Oi%4Rr0guBK^{Bz3!8 zD5C#~Auw#3=3p@B^?I#V3tFW)WkiysZQFyvKoA57)fv$~7a&-!>&mjs^L(q-f>!Cq zgF>SihT&*5>U27yD5|O&ZF+7G2s(US*Ck16G#Z^w2Yc;COpfEIswxNq#Kh##cT0w# zx=E4*@vkU~ZQHZ=g}2bdcRC#q(2Wr~FCg5Ii=rrs;%GFQ+L%uM#Wc-+zb}fSEX(sa zdR{;vBUM#-umwv@C1tbQ_y39mg?EQ1&cEq0II&?EFeBgiaTtd9it_IR!m(pnmSGrp;GaPcEXdmb z7lxp(RaI5jb(pWG&h3~>P19`KUI@*?E;9^6RaHe%bX~tL8$pOckFZdD-3Abhw%u+Q z-;m%ND$_Kvv*5xBh@3x1UpofPW=NLhcDvnbwN_VGIgU#t5X}8?2qSCTo+ur4d1^NOdO9fQ~Wz6hhCa+JzA7(2AfNpmWfRQ!>NN z1bj7obDM#-+~#a;ZFF?Nf5eQIyW1M;I~p6TFW5WQTdVJ{tz(t~9cpi_Z*DAaZnHNq z9iTB<+TO9LrP12%Y_T_#Wi4%LwmDmzZPumwr@XECnR;u*vI@OjMW{@f%E0xRtGG_t zt^X}B*Ut70^)|bs*;+X@7AQAeO_|b_Z*#r~IUi)kJJrk}u<=-%UKvRZ;_NA5Ds&zSXX`(8+z$31<4J?Hvscp;adadw;uf%= zdi)8RNik70RV5VD%hK_hYz*~*v#Jh+DFy6Ac*HXW&7p))o?a$rTW4`V$j@y(qfkmT zIT{{L5(Z>iNF&3qEj5BY0GpP=|P{JY8^;8%y5c*x%6$8M*LIam^1B4MKaZeY6m2Eyg7zG|o zMR`oo-g5hkJ+6pKB)>-xT!JsGvY`{$?HdOI1C_M!myRW7 zmcs}(&42WbQ)3BL)U%y%4~yDCHlM^|!80Oiu;y_R|8?8)JPyNYBl3Bb&mOuMk%B&! z%b?dUQF^eD!{+M*!R>>j@Kln|?^<1G#6S|orkSg^eLE5wX7!=CFHLHCs-3l?y23Li zv89YF!oD?gj4*-$NnB>>LvQ}#7I;DNCe1Q&t_gU7V8T5d(V*f6yVqJX;9NAK5ax2r z>i7TD2MKq>L@D9OnDfDjAvc6G7?ddue!k(x91g%H6vl>vW5_>jlz<1M0VE2 z^iU$6(HK$sUaKlH0M^WQf@x0G^G8qj3%(H005<5XlY@hdcxXZy|K#a;CX8_=8o^xF z!iRUBI4@j|0VhC-vrdcXwI1~bVt;lm&*OnJ6F6grf}0%Ozj}hIzOK|+N8)2_C1Rf8 z;HPV^1xAz&xVU-l1FwJYg~=MJ|=752jeFb^XQ14a{J zNWnF5+_8`amYbxh-F-%cay3wh9%|9=6WTGp%_WVl6zk|8Bnem zB{lqWS52uI%|?lwijL2Q*?E(sl@$Nioh!j@Hj)gBY)8GYVPLDXX_LgIo~A`+GzUtw zpV0FOsO7O?d)k(c!ZtadAVA+YSnrMlkBQ3WS>Uhqw zs{PRUNxA>No1=25Z_m1A`5Y=ify^b3bbTd^e_g(yHz<6$rFuaYW@=;ciiS77={>w^ z@!`I2-)g8VHZk=$&MLjrw*8|c+jo3?bm#giYc7viRi>|)^c9o7V$xSk`ie Writting Process + +
    +
  • See my stats
    • +
  • Manage my data
    • +
diff --git a/extension/writing.js b/extension/writing.js new file mode 100644 index 000000000..be67b6186 --- /dev/null +++ b/extension/writing.js @@ -0,0 +1,56 @@ +var WRITINGJS_SERVER = "https://test.mitros.org/webapi/"; + +function writingjs_ajax(data) { + httpRequest = new XMLHttpRequest(); + //httpRequest.withCredentials = true; + httpRequest.open("POST", WRITINGJS_SERVER); + httpRequest.send(JSON.stringify(data)); +} + +document.body.style.border = "5px solid blue"; + +function writing_eventlistener(event) { + var event_data = {}; + event_data["event_type"] = "keypress"; + properties = ['altKey', 'charCode', 'code', 'ctrlKey', 'isComposing', 'key', 'keyCode', 'location', 'metaKey', 'repeat', 'shiftKey', 'which', 'isTrusted', 'timeStemp', 'type']; + for (var property in properties) { + event_data[properties[property]] = event[properties[property]]; + } + event_data['date'] = new Date().toLocaleString('en-US') + console.log(JSON.stringify(event_data)); + writingjs_ajax(event_data); +} + +document.addEventListener("keypress", writing_eventlistener); +document.addEventListener("keydown", writing_eventlistener); +document.addEventListener("keyup", writing_eventlistener); + +var iframes = document.getElementsByTagName("iframe") +for(iframe in iframes){ + if(iframes[iframe].contentDocument) { + console.log(iframes[iframe].contentDocument); + iframes[iframe].contentDocument.addEventListener("keypress", writing_eventlistener); + } +} +/*chrome.webRequest.onBeforeRequest.addListener( + function(request) { + if (request.url.indexOf('/save?') != -1) { + // Regexp and general theme of code based on + // http://features.jsomers.net/how-i-reverse-engineered-google-docs/ + var docId = request.url.match("docs\.google\.com\/document\/d\/(.*?)\/save")[1] + + var data = { + "bundles": request.body.formData.bundles, + "revNo": request.body.formData.rev, + "docId": docId, + "timeStamp" : request.timeStamp + } + + writingjs_ajax(data); + //draftback.addPendingRevision(data, request.requestId) + } + }, + { urls: ["*://docs.google.com/*"] }, + ['requestBody'] +) +*/ From f8b24fa51280cbb10ed3e5af936241508d078343 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Mon, 26 Aug 2019 06:07:50 -0400 Subject: [PATCH 002/750] Basic monitoring of gmail --- .gitignore | 1 + extension/background.js | 49 +++++++++++++++++++++++++++++++++++++---- extension/manifest.json | 3 ++- extension/writing.js | 9 +++++++- 4 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..b25c15b81 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*~ diff --git a/extension/background.js b/extension/background.js index 0760f4247..b79fee9d1 100644 --- a/extension/background.js +++ b/extension/background.js @@ -1,6 +1,36 @@ +/* +Background script. This works across all of Google Chrome. +*/ + var WRITINGJS_SERVER = "https://test.mitros.org/webapi/"; +function gmail_text() { + /* + This function returns all the editable text in the current gmail + window. Note that in a threaded discussion, it's possible to + have several open on the same page. + + This is brittle; Google may change implementation and this will + break. + */ + var documents = document.getElementsByClassName("editable"); + for(document in documents) { + documents[document] = documents[document].innerHTML; + } + return documents; +} + function writingjs_ajax(data) { + /* + Helper function to send a logging AJAX request to the server. + This function takes a JSON dictionary of data. + + TODO: Convert to a queue for offline operation using Chrome + Storage API? Cache to Chrome Storage? Chrome Storage doesn't + support meaningful concurrency, + + TODO: Abstract out into a common function, + */ httpRequest = new XMLHttpRequest(); //httpRequest.withCredentials = true; httpRequest.open("POST", WRITINGJS_SERVER); @@ -8,26 +38,37 @@ function writingjs_ajax(data) { } function writing_eventlistener(event) { + /* + Here, we process keystroke events and send them to the server. We want fine-grained writing data + with timing. + */ var event_data = {}; event_data["event_type"] = "keypress"; properties = ['altKey', 'charCode', 'code', 'ctrlKey', 'isComposing', 'key', 'keyCode', 'location', 'metaKey', 'repeat', 'shiftKey', 'which', 'isTrusted', 'timeStemp', 'type']; for (var property in properties) { event_data[properties[property]] = event[properties[property]]; } - event_data['date'] = new Date().toLocaleString('en-US') + event_data['date'] = new Date().toLocaleString('en-US'); console.log(JSON.stringify(event_data)); writingjs_ajax(event_data); } chrome.webRequest.onBeforeRequest.addListener( function(request) { + var formdata = {}; + if(request.requestBody) { + formdata = request.requestBody.formData; + } + if(!formdata) { + formdata = {}; + } writingjs_ajax({ 'event_type':'request', 'url': request.url, - 'from_data': request.requestBody.formData + 'from_data': formdata }); }, - { urls: ["*://docs.google.com/*"] }, + { urls: ["*://docs.google.com/*"/*, "*://mail.google.com/*"*/] }, ['requestBody'] ) @@ -35,4 +76,4 @@ chrome.webRequest.onBeforeRequest.addListener( code: 'console.log("addd")' });*/ -writingjs_ajax({"Loaded": true}); +writingjs_ajax({"Loaded now": true}); diff --git a/extension/manifest.json b/extension/manifest.json index 7c5d9d9d9..406bfadf6 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -19,7 +19,7 @@ "content_scripts": [ { - "matches": ["*://docs.google.com/*", "*://*.mozilla.org/*"], + "matches": ["*://docs.google.com/*", "*://*.mozilla.org/*", "*://mail.google.com/*"], "js": ["writing.js"] }], "background": { @@ -28,6 +28,7 @@ "permissions": [ "webRequest", "*://docs.google.com/*", + "*://mail.google.com/*", "clipboardRead" ] } diff --git a/extension/writing.js b/extension/writing.js index be67b6186..c392b8c8a 100644 --- a/extension/writing.js +++ b/extension/writing.js @@ -1,3 +1,7 @@ +/* +Page script. This is injected into each web page on associated web sites. +*/ + var WRITINGJS_SERVER = "https://test.mitros.org/webapi/"; function writingjs_ajax(data) { @@ -16,7 +20,10 @@ function writing_eventlistener(event) { for (var property in properties) { event_data[properties[property]] = event[properties[property]]; } - event_data['date'] = new Date().toLocaleString('en-US') + event_data['date'] = new Date().toLocaleString('en-US'); + event_data['url'] = window.location.href; + console.log(event_data['url']); + console.log(JSON.stringify(event_data)); writingjs_ajax(event_data); } From 95f7d6ea0d03278fc5992db6a4e1b52c277500d2 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Mon, 26 Aug 2019 15:48:34 -0400 Subject: [PATCH 003/750] Google Docs works pretty well. --- extension/background.js | 88 +++++++++++++++++++-------------- extension/manifest.json | 13 +++-- extension/writing.js | 98 +++++++++++++++++++++++++------------ extension/writing_common.js | 33 +++++++++++++ 4 files changed, 158 insertions(+), 74 deletions(-) create mode 100644 extension/writing_common.js diff --git a/extension/background.js b/extension/background.js index b79fee9d1..39f3d3d9a 100644 --- a/extension/background.js +++ b/extension/background.js @@ -2,40 +2,18 @@ Background script. This works across all of Google Chrome. */ -var WRITINGJS_SERVER = "https://test.mitros.org/webapi/"; - -function gmail_text() { - /* - This function returns all the editable text in the current gmail - window. Note that in a threaded discussion, it's possible to - have several open on the same page. - - This is brittle; Google may change implementation and this will - break. - */ - var documents = document.getElementsByClassName("editable"); - for(document in documents) { - documents[document] = documents[document].innerHTML; +function this_a_google_docs_save(request) { + /* + Check if this is a Google Docs save request. Return true for something like: + https://docs.google.com/document/d/1lt_lSfEM9jd7Ga6uzENS_s8ZajcxpE0cKuzXbDoBoyU/save?id=dfhjklhsjklsdhjklsdhjksdhkjlsdhkjsdhsdkjlhsd&sid=dhsjklhsdjkhsdas&vc=2&c=2&w=2&smv=2&token=lasjklhasjkhsajkhsajkhasjkashjkasaajhsjkashsajksas&includes_info_params=true + And false otherwise + */ + if(request.url.match(/.*:\/\/docs\.google\.com\/document\/d\/([^\/]*)\/save/i)) { + return true; } - return documents; + return false; } -function writingjs_ajax(data) { - /* - Helper function to send a logging AJAX request to the server. - This function takes a JSON dictionary of data. - - TODO: Convert to a queue for offline operation using Chrome - Storage API? Cache to Chrome Storage? Chrome Storage doesn't - support meaningful concurrency, - - TODO: Abstract out into a common function, - */ - httpRequest = new XMLHttpRequest(); - //httpRequest.withCredentials = true; - httpRequest.open("POST", WRITINGJS_SERVER); - httpRequest.send(JSON.stringify(data)); -} function writing_eventlistener(event) { /* @@ -53,7 +31,34 @@ function writing_eventlistener(event) { writingjs_ajax(event_data); } +var RAW_DEBUG = false; // Do not save debug requests. We flip this frequently. Perhaps this should be a cookie or browser.storage? + chrome.webRequest.onBeforeRequest.addListener( + /* + This allows us to log web requests. There are two types of web requests: + * Ones we understand (SEMANTIC) + * Ones we don't (RAW/DEBUG) + + There is an open question as to how we ought to handle RAW/DEBUG + events. We will reduce potential issues around collecting data + we don't want (privacy, storage, bandwidth) if we silently drop + these. On the other hand, we significantly increase risk of + losing user data should Google ever change their web API. If we + log everything, we have good odds of being able to + reverse-engineer the new API, and reconstruct what happened. + + Our current strategy is to: + * Log the former requests in a clean way, extracting the data we + want + * Have a flag to log the debug requests (which includes the + unparsed version of events we want). + We should step through and see how this code manages failures, + + For development purposes, both modes of operation are + helpful. Having these is nice for reverse-engineering, + especially new pages. They do inject a lot of noise, though, and + from there, being able to easily ignore these is nice. + */ function(request) { var formdata = {}; if(request.requestBody) { @@ -62,11 +67,22 @@ chrome.webRequest.onBeforeRequest.addListener( if(!formdata) { formdata = {}; } - writingjs_ajax({ - 'event_type':'request', - 'url': request.url, - 'from_data': formdata - }); + if(RAW_DEBUG) { + writingjs_ajax({ + 'event_type': 'raw_request', + 'url': request.url, + 'form_data': formdata + }); + } + if(this_a_google_docs_save(request)) { + writingjs_ajax({ + 'event_type': 'google_docs_save', + 'doc_id': googledocs_id_from_url(request.url), + 'bundles': JSON.parse(formdata.bundles), + 'rev': formdata.rev, + 'timestamp': parseInt(request.timeStamp, 10) + }); + } }, { urls: ["*://docs.google.com/*"/*, "*://mail.google.com/*"*/] }, ['requestBody'] diff --git a/extension/manifest.json b/extension/manifest.json index 406bfadf6..1d0f2d56d 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -17,14 +17,13 @@ } }, - "content_scripts": [ - { - "matches": ["*://docs.google.com/*", "*://*.mozilla.org/*", "*://mail.google.com/*"], - "js": ["writing.js"] - }], + "content_scripts": [ { + "matches": ["*://docs.google.com/*", "*://*.mozilla.org/*", "*://mail.google.com/*"], + "js": ["writing_common.js", "writing.js"] + }], "background": { - "scripts": ["background.js"] - }, + "scripts": ["writing_common.js", "background.js"] + }, "permissions": [ "webRequest", "*://docs.google.com/*", diff --git a/extension/writing.js b/extension/writing.js index c392b8c8a..ff6acda5b 100644 --- a/extension/writing.js +++ b/extension/writing.js @@ -2,17 +2,12 @@ Page script. This is injected into each web page on associated web sites. */ -var WRITINGJS_SERVER = "https://test.mitros.org/webapi/"; +document.body.style.border = "5px solid blue"; -function writingjs_ajax(data) { - httpRequest = new XMLHttpRequest(); - //httpRequest.withCredentials = true; - httpRequest.open("POST", WRITINGJS_SERVER); - httpRequest.send(JSON.stringify(data)); +function doc_id() { + return googledocs_id_from_url(window.location.href); } -document.body.style.border = "5px solid blue"; - function writing_eventlistener(event) { var event_data = {}; event_data["event_type"] = "keypress"; @@ -21,13 +16,15 @@ function writing_eventlistener(event) { event_data[properties[property]] = event[properties[property]]; } event_data['date'] = new Date().toLocaleString('en-US'); - event_data['url'] = window.location.href; + event_data['id'] = doc_id(); console.log(event_data['url']); console.log(JSON.stringify(event_data)); writingjs_ajax(event_data); } + + document.addEventListener("keypress", writing_eventlistener); document.addEventListener("keydown", writing_eventlistener); document.addEventListener("keyup", writing_eventlistener); @@ -37,27 +34,66 @@ for(iframe in iframes){ if(iframes[iframe].contentDocument) { console.log(iframes[iframe].contentDocument); iframes[iframe].contentDocument.addEventListener("keypress", writing_eventlistener); + iframes[iframe].contentDocument.addEventListener("keydown", writing_eventlistener); + iframes[iframe].contentDocument.addEventListener("keyup", writing_eventlistener); } } -/*chrome.webRequest.onBeforeRequest.addListener( - function(request) { - if (request.url.indexOf('/save?') != -1) { - // Regexp and general theme of code based on - // http://features.jsomers.net/how-i-reverse-engineered-google-docs/ - var docId = request.url.match("docs\.google\.com\/document\/d\/(.*?)\/save")[1] - - var data = { - "bundles": request.body.formData.bundles, - "revNo": request.body.formData.rev, - "docId": docId, - "timeStamp" : request.timeStamp - } - - writingjs_ajax(data); - //draftback.addPendingRevision(data, request.requestId) - } - }, - { urls: ["*://docs.google.com/*"] }, - ['requestBody'] -) -*/ + + +function gmail_text() { + /* + This function returns all the editable text in the current gmail + window. Note that in a threaded discussion, it's possible to + have several open on the same page. + + This is brittle; Google may change implementation and this will + break. + */ + var documents = document.getElementsByClassName("editable"); + for(document in documents) { + documents[document] = { + 'text': documents[document].innerHTML + }; + } + return documents; +} + +function google_docs_title() { + /* + Return the title of a Google Docs document + */ + return document.getElementsByClassName("docs-title-input")[0].value; +} + +function google_docs_partial_text() { + /* + Return the *loaded* text of a Google Doc. Note that for long + documents, this may not be the *complete* text since off-screen + pages may be lazy-loaded. The text omits formatting, which is + helpful for many types of analysis + */ + return document.getElementsByClassName("kix-page")[0].innerText; +} + +function google_docs_partial_html() { + /* + Return the *loaded* HTML of a Google Doc. Note that for long + documents, this may not be the *complete* HTML, since off-screen + pages may be lazy-loaded. This includes HTML formatting, which + may be helpful, but is incredibly messy. + + I hate Google's HTML. What's wrong with simple, clean, semantic + tags? Why do we need something like this instead: + + Seriously, Google? + */ + return document.getElementsByClassName("kix-page")[0].innerHTML; +} + +writingjs_ajax({ + "event_type": "Google Docs loaded", + "partial_text": google_docs_partial_text(), +// "partial_html": google_docs_partial_html(), + "title": google_docs_title(), + "id": doc_id +}) diff --git a/extension/writing_common.js b/extension/writing_common.js new file mode 100644 index 000000000..aa5d053ae --- /dev/null +++ b/extension/writing_common.js @@ -0,0 +1,33 @@ +var WRITINGJS_SERVER = "https://test.mitros.org/webapi/"; + +function writingjs_ajax(data) { + /* + Helper function to send a logging AJAX request to the server. + This function takes a JSON dictionary of data. + + TODO: Convert to a queue for offline operation using Chrome + Storage API? Cache to Chrome Storage? Chrome Storage doesn't + support meaningful concurrency, + */ + + httpRequest = new XMLHttpRequest(); + //httpRequest.withCredentials = true; + httpRequest.open("POST", WRITINGJS_SERVER); + httpRequest.send(JSON.stringify(data)); +} + + +function googledocs_id_from_url(url) { + /* + Given a URL like: + https://docs.google.com/document/d/jkldfhjklhdkljer8934789468976sduiyui34778dey/edit/foo/bar + extract the associated document ID: + jkldfhjklhdkljer8934789468976sduiyui34778dey + Return null if not a valid URL + */ + var match = url.match(/.*:\/\/docs\.google\.com\/document\/d\/([^\/]*)\/.*/i); + if(match) { + return match[1]; + } + return null; +} From ac31c0a649459309fc773ff6d045b0ad7d534619 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Tue, 27 Aug 2019 14:16:44 -0400 Subject: [PATCH 004/750] Starting to poke at key generation --- extension/3rdparty/sha256.js | 27 +++++++++++ extension/background.js | 4 ++ extension/manifest.json | 7 ++- extension/writing.js | 91 ++++++++++++++++++++++++++++++++---- extension/writing_common.js | 1 + 5 files changed, 120 insertions(+), 10 deletions(-) create mode 100644 extension/3rdparty/sha256.js diff --git a/extension/3rdparty/sha256.js b/extension/3rdparty/sha256.js new file mode 100644 index 000000000..e62bf9e52 --- /dev/null +++ b/extension/3rdparty/sha256.js @@ -0,0 +1,27 @@ +/* + A JavaScript implementation of the SHA family of hashes, as + defined in FIPS PUB 180-4 and FIPS PUB 202, as well as the corresponding + HMAC implementation as defined in FIPS PUB 198a + + Copyright 2008-2018 Brian Turek, 1998-2009 Paul Johnston & Contributors + Distributed under the BSD License + See http://caligatio.github.com/jsSHA/ for more information +*/ +'use strict';(function(I){function w(c,a,d){var l=0,b=[],g=0,f,n,k,e,h,q,y,p,m=!1,t=[],r=[],u,z=!1;d=d||{};f=d.encoding||"UTF8";u=d.numRounds||1;if(u!==parseInt(u,10)||1>u)throw Error("numRounds must a integer >= 1");if(0===c.lastIndexOf("SHA-",0))if(q=function(b,a){return A(b,a,c)},y=function(b,a,l,f){var g,e;if("SHA-224"===c||"SHA-256"===c)g=(a+65>>>9<<4)+15,e=16;else throw Error("Unexpected error in SHA-2 implementation");for(;b.length<=g;)b.push(0);b[a>>>5]|=128<<24-a%32;a=a+l;b[g]=a&4294967295; +b[g-1]=a/4294967296|0;l=b.length;for(a=0;a>>3;g=e/4-1;if(eb/8){for(;a.length<=g;)a.push(0);a[g]&=4294967040}for(b=0;b<=g;b+=1)t[b]=a[b]^909522486,r[b]=a[b]^1549556828;n=q(t,n);l=h;m=!0};this.update=function(a){var c,f,e,d=0,p=h>>>5;c=k(a,b,g);a=c.binLen;f=c.value;c=a>>>5;for(e=0;e>> +5);g=a%h;z=!0};this.getHash=function(a,f){var d,h,k,q;if(!0===m)throw Error("Cannot call getHash after setting HMAC key");k=C(f);switch(a){case "HEX":d=function(a){return D(a,e,k)};break;case "B64":d=function(a){return E(a,e,k)};break;case "BYTES":d=function(a){return F(a,e)};break;case "ARRAYBUFFER":try{h=new ArrayBuffer(0)}catch(v){throw Error("ARRAYBUFFER not supported by this environment");}d=function(a){return G(a,e)};break;default:throw Error("format must be HEX, B64, BYTES, or ARRAYBUFFER"); +}q=y(b.slice(),g,l,p(n));for(h=1;h>>2]>>>8*(3+b%4*-1),l+="0123456789abcdef".charAt(g>>>4&15)+"0123456789abcdef".charAt(g&15);return d.outputUpper?l.toUpperCase():l}function E(c,a,d){var l="",b=a/8,g,f,n;for(g=0;g>>2]:0,n=g+2>>2]:0,n=(c[g>>>2]>>>8*(3+g%4*-1)&255)<<16|(f>>>8*(3+(g+1)%4*-1)&255)<<8|n>>>8*(3+(g+2)%4*-1)&255,f=0;4>f;f+=1)8*g+6*f<=a?l+="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".charAt(n>>> +6*(3-f)&63):l+=d.b64Pad;return l}function F(c,a){var d="",l=a/8,b,g;for(b=0;b>>2]>>>8*(3+b%4*-1)&255,d+=String.fromCharCode(g);return d}function G(c,a){var d=a/8,l,b=new ArrayBuffer(d),g;g=new Uint8Array(b);for(l=0;l>>2]>>>8*(3+l%4*-1)&255;return b}function C(c){var a={outputUpper:!1,b64Pad:"=",shakeLen:-1};c=c||{};a.outputUpper=c.outputUpper||!1;!0===c.hasOwnProperty("b64Pad")&&(a.b64Pad=c.b64Pad);if("boolean"!==typeof a.outputUpper)throw Error("Invalid outputUpper formatting option"); +if("string"!==typeof a.b64Pad)throw Error("Invalid b64Pad formatting option");return a}function B(c,a){var d;switch(a){case "UTF8":case "UTF16BE":case "UTF16LE":break;default:throw Error("encoding must be UTF8, UTF16BE, or UTF16LE");}switch(c){case "HEX":d=function(a,b,c){var f=a.length,d,k,e,h,q;if(0!==f%2)throw Error("String of HEX type must be in byte increments");b=b||[0];c=c||0;q=c>>>3;for(d=0;d>>1)+q;for(e=h>>>2;b.length<=e;)b.push(0);b[e]|=k<<8*(3+h%4*-1)}return{value:b,binLen:4*f+c}};break;case "TEXT":d=function(c,b,d){var f,n,k=0,e,h,q,m,p,r;b=b||[0];d=d||0;q=d>>>3;if("UTF8"===a)for(r=3,e=0;ef?n.push(f):2048>f?(n.push(192|f>>>6),n.push(128|f&63)):55296>f||57344<=f?n.push(224|f>>>12,128|f>>>6&63,128|f&63):(e+=1,f=65536+((f&1023)<<10|c.charCodeAt(e)&1023),n.push(240|f>>>18,128|f>>>12&63,128|f>>>6&63,128|f&63)),h=0;h>>2;b.length<=m;)b.push(0);b[m]|=n[h]<<8*(r+p%4*-1);k+=1}else if("UTF16BE"===a||"UTF16LE"===a)for(r=2,n="UTF16LE"===a&&!0||"UTF16LE"!==a&&!1,e=0;e>>8);p=k+q;for(m=p>>>2;b.length<=m;)b.push(0);b[m]|=f<<8*(r+p%4*-1);k+=2}return{value:b,binLen:8*k+d}};break;case "B64":d=function(a,b,c){var f=0,d,k,e,h,q,m,p;if(-1===a.search(/^[a-zA-Z0-9=+\/]+$/))throw Error("Invalid character in base-64 string");k=a.indexOf("=");a=a.replace(/\=/g, +"");if(-1!==k&&k Date: Tue, 27 Aug 2019 14:28:11 -0400 Subject: [PATCH 005/750] Popup works! --- extension/manifest.json | 8 +++++--- extension/popup/action.css | 4 ++++ extension/popup/settings.html | 8 ++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 extension/popup/action.css diff --git a/extension/manifest.json b/extension/manifest.json index 46e562f25..335db4a4d 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -8,8 +8,7 @@ "version": "1.0", "description": "Tracks writing in Google Docs, and provides nifty insights to you and your teachers!", - "page_action": { - "browser_style": true, + "browser_action": { "default_title": "Writing Process", "default_popup": "popup/settings.html", "default_icon": { @@ -32,5 +31,8 @@ "*://mail.google.com/*", "clipboardRead", "storage" - ] + ], + "icons": { + "48": "icons/lousy-fountain-pen-48.png" + } } diff --git a/extension/popup/action.css b/extension/popup/action.css new file mode 100644 index 000000000..59eb8157e --- /dev/null +++ b/extension/popup/action.css @@ -0,0 +1,4 @@ +html, body { + width: 400px; +} + diff --git a/extension/popup/settings.html b/extension/popup/settings.html index f57d0ff43..fda3046f9 100644 --- a/extension/popup/settings.html +++ b/extension/popup/settings.html @@ -1,6 +1,14 @@ + + + + + +

Writting Process

  • See my stats
  • Manage my data
+ + From 467b972fd7bd1596dcdf28530fce024a59631e0e Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Thu, 5 Sep 2019 15:10:02 -0400 Subject: [PATCH 006/750] Script to start reconstructing from events --- reconstruct/reconstruct.py | 37 +++++++++++++++++++++++++++++++++++++ requirements.txt | 3 +++ 2 files changed, 40 insertions(+) create mode 100644 reconstruct/reconstruct.py create mode 100644 requirements.txt diff --git a/reconstruct/reconstruct.py b/reconstruct/reconstruct.py new file mode 100644 index 000000000..91dfab5a0 --- /dev/null +++ b/reconstruct/reconstruct.py @@ -0,0 +1,37 @@ +import json +import pandas + +js = json.load(open("chunked3.json")) + +document = " " + +def apply_changes(document, changes): + for change_line in changes: + #if 'mts' in change[0]: + # del change[0]['mts'] + #print(change_line) + if isinstance(change_line, list): + change = change_line[0] + else: + change = change_line + + # Insert + if change['ty'] == "is": + document = document[:change['ibi']]+change['s']+document[change['ibi']:] + # Multiple changes clumped together + elif change['ty'] == "mlti": + #print(change) + document = apply_changes(document, change['mts']) + # Delete from si to ei + elif change['ty'] == "ds": + document = document[:change["si"]]+document[change["ei"]:] + # This formats text. We can ignore this for now. + elif change['ty'] == "as": + pass + #print(change) + else: + raise Exception("Unknown change") + return document + +document = apply_changes(" ", js['changelog']) +print(document) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..d8864a5d5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +pandas +tornado +ipython From 1404ca70d8a516f3e1fbc14f5a960e76e1c99518 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Fri, 6 Sep 2019 15:41:23 -0400 Subject: [PATCH 007/750] asyncpg --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index d8864a5d5..39731981e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ pandas tornado ipython +asyncpg + From 253f4dbb9d71daf56c0ebcba8ef1cee527a2cc0c Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Wed, 2 Oct 2019 14:28:21 +0100 Subject: [PATCH 008/750] Initial webapp commit. About to refactor SQL --- webapp/init.sql | 124 +++++++++++++++++++++++++++++++++++++++++++++++ webapp/webapp.py | 47 ++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 webapp/init.sql create mode 100644 webapp/webapp.py diff --git a/webapp/init.sql b/webapp/init.sql new file mode 100644 index 000000000..17c40e285 --- /dev/null +++ b/webapp/init.sql @@ -0,0 +1,124 @@ +reset: | + DROP TABLE IF EXISTS USERS; + DROP TABLE IF EXISTS WRITING_DELTAS; + DROP TABLE IF EXISTS DOCUMENTS; + DROP FUNCTION IF EXISTS insert_writing_delta; + +init: | + CREATE TABLE USERS ( + idx SERIAL PRIMARY KEY, + username text UNIQUE, + email text, + date_created timestamp NOT NULL DEFAULT NOW() + ); + + CREATE TABLE IF NOT EXISTS DOCUMENTS ( + idx SERIAL PRIMARY KEY, + docstring char(48) UNIQUE, + date_created timestamp NOT NULL DEFAULT NOW() + ); + + CREATE TABLE IF NOT EXISTS WRITING_DELTAS ( + idx SERIAL PRIMARY KEY, + user_id integer REFERENCES USERS (idx), -- Who is editing? + document integer REFERENCES DOCUMENTS (idx), -- Which document? + date_created timestamp NOT NULL DEFAULT NOW(), + ty char(2), -- Type of edit. Insert, delete, alter + si integer, -- Start index for deletion + ei integer, -- End index for deletion + ibi integer, -- Insert index + s text, -- Text to insert + ft text -- For debugging: Ongoing reconstruction of full text + ); + + CREATE OR REPLACE FUNCTION insert_writing_delta( + gusername text, + gdocstring char(48), + ty char(2), -- Type of edit. Insert, delete, alter + si integer, -- Start index for deletion + ei integer, -- End index for deletion + ibi integer, -- Insert index + s text, -- Text to insert + ft text -- For debugging: Ongoing reconstruction of full text + ) RETURNS text + LANGUAGE plpgsql + AS $$ + DECLARE + strresult text; + affected_rows integer; + BEGIN + strresult := ''; + -- If the user does not exist, create the user. Add 'New User' to the return value + if NOT EXISTS (SELECT 1 FROM USERS where USERS.username = gusername) THEN + strresult := strresult || '[New User]'; + INSERT INTO USERS (username) VALUES (gusername); + END IF; + + -- If the document does not exist, create the document. Add "New Document" to the return value + if NOT EXISTS (SELECT 1 FROM DOCUMENTS where DOCUMENTS.docstring = gdocstring) THEN + strresult := strresult || '[New Document]'; + INSERT INTO DOCUMENTS (docstring) VALUES (gdocstring); + END IF; + -- Insert the writing delta into the database + with INSERT_ROW_COUNT as + (INSERT INTO WRITING_DELTAS + (user_id, document, ty, si, ei, ibi, s, ft) + (SELECT + users.idx, documents.idx, ty, si, ei, ibi, s, ft + FROM + users, documents where users.username=gusername and documents.docstring=gdocstring) + RETURNING 1) + SELECT COUNT(*) INTO affected_rows FROM INSERT_ROW_COUNT; + + -- This is a little bit awkward, but we return: + -- 1. Number of rows inserted + -- 2. Whether a new user or document was created + -- As a string. + return cast(affected_rows as varchar) || ' ' || strresult; + COMMIT; + END; + $$; + -- Example: SELECT insert_writing_delta('pmitros', 'random-google-doc-id', 'is', 7,8,4,'hello','temp'); + + CREATE OR REPLACE FUNCTION load_writing_deltas( + gusername text, + gdocstring char(48) + ) RETURNS refcursor + LANGUAGE plpsgwwql + AS $$ + DECLARE + ref refcursor; + BEGIN + OPEN ref FOR + SELECT + WRITING_DELTAS.idx, WRITING_DELTAS.date_created, ty, si, ei, ibi, s, ft + FROM + WRITING_DELTAS, USERS, DOCUMENTS + WHERE + WRITING_DELTAS.user_id = USERS.idx AND + WRITING_DELTAS.document = DOCUMENTS.idx AND + DOCUMENTS.docstring = gdocstring AND + USERS.username = gusername; + return ref; + END; + $$; + + +insert_writing_delta: | + -- PREPARE insert_into_WRITING_DELTAS (username, document, ty, si, ei, ipi, s, ft) AS + -- INSERT INTO WRITING_DELTAS + -- (ty, si, ei, ipi, s, ft) + -- VALUES + -- ($1, $2, $3, $4, $5, $6); + +fetch_writing_deltas : | + PREPARE fetch_writing_deltas (gusername, gdocument) AS + SELECT + WRITING_DELTAS.idx, WRITING_DELTAS.date_created, ty, si, ei, ibi, s, ft + FROM + WRITING_DELTAS, USERS, DOCUMENTS + WHERE + WRITING_DELTAS.user_id = USERS.idx AND + WRITING_DELTAS.document = DOCUMENTS.idx AND + DOCUMENTS.docstring = gdocstring AND + USERS.username = gusername; diff --git a/webapp/webapp.py b/webapp/webapp.py new file mode 100644 index 000000000..b68e3f59d --- /dev/null +++ b/webapp/webapp.py @@ -0,0 +1,47 @@ +import tornado.ioloop +import tornado.web + +import asyncio +import asyncpg + +def initialize_database(): + conn=asyncpg.connect() + await conn.execute(open("init.sql").read()) + +def add_record(idx, ty, si, ei, ibi, s, ft): + pass + +class MainHandler(tornado.web.RequestHandler): + def set_default_headers(self): + self.set_header("Access-Control-Allow-Origin", "*") + self.set_header("Access-Control-Allow-Headers", "x-requested-with") + self.set_header('Access-Control-Allow-Methods', 'POST, GET, OPTIONS, DELETE') + self.set_header('Access-Control-Allow-Credentials', 'True') + + def get(self, url): + self.set_header("Content-Type", "text/plain") + self.write("okay") + print(url) + print(self.request.body) + + def post(self, url): + self.write("okay") + print(url) + print(self.request.body) + + def options(self): + # no body + print("CORS Request!") + self.set_status(204) + self.finish() + +def make_app(): + return tornado.web.Application([ + (r"/(.*)", MainHandler), + ]) + +if __name__ == "__main__": + conn = initialize_database() + app = make_app() + app.listen(8888) + tornado.ioloop.IOLoop.current().start() From fb090832e5c34875edb2592e8e36b0a495a753d7 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Wed, 2 Oct 2019 20:58:57 +0100 Subject: [PATCH 009/750] First working version DB connection. Poor schema.... --- requirements.txt | 2 +- webapp/init.sql | 64 +++++++++++++++--------------------------------- webapp/orm.py | 47 +++++++++++++++++++++++++++++++++++ webapp/webapp.py | 9 ++----- 4 files changed, 70 insertions(+), 52 deletions(-) create mode 100644 webapp/orm.py diff --git a/requirements.txt b/requirements.txt index 39731981e..b1fd9edc3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ pandas tornado ipython asyncpg - +yaml diff --git a/webapp/init.sql b/webapp/init.sql index 17c40e285..6b0749293 100644 --- a/webapp/init.sql +++ b/webapp/init.sql @@ -5,7 +5,7 @@ reset: | DROP FUNCTION IF EXISTS insert_writing_delta; init: | - CREATE TABLE USERS ( + CREATE TABLE IF NOT EXISTS USERS ( idx SERIAL PRIMARY KEY, username text UNIQUE, email text, @@ -21,7 +21,7 @@ init: | CREATE TABLE IF NOT EXISTS WRITING_DELTAS ( idx SERIAL PRIMARY KEY, user_id integer REFERENCES USERS (idx), -- Who is editing? - document integer REFERENCES DOCUMENTS (idx), -- Which document? + document integer REFERENCES DOCUMENTS (idx), -- Which document? date_created timestamp NOT NULL DEFAULT NOW(), ty char(2), -- Type of edit. Insert, delete, alter si integer, -- Start index for deletion @@ -67,7 +67,7 @@ init: | users.idx, documents.idx, ty, si, ei, ibi, s, ft FROM users, documents where users.username=gusername and documents.docstring=gdocstring) - RETURNING 1) + RETURNING 1) SELECT COUNT(*) INTO affected_rows FROM INSERT_ROW_COUNT; -- This is a little bit awkward, but we return: @@ -79,46 +79,22 @@ init: | END; $$; -- Example: SELECT insert_writing_delta('pmitros', 'random-google-doc-id', 'is', 7,8,4,'hello','temp'); - - CREATE OR REPLACE FUNCTION load_writing_deltas( - gusername text, - gdocstring char(48) - ) RETURNS refcursor - LANGUAGE plpsgwwql - AS $$ - DECLARE - ref refcursor; - BEGIN - OPEN ref FOR - SELECT - WRITING_DELTAS.idx, WRITING_DELTAS.date_created, ty, si, ei, ibi, s, ft - FROM - WRITING_DELTAS, USERS, DOCUMENTS - WHERE - WRITING_DELTAS.user_id = USERS.idx AND - WRITING_DELTAS.document = DOCUMENTS.idx AND - DOCUMENTS.docstring = gdocstring AND - USERS.username = gusername; - return ref; - END; - $$; - -insert_writing_delta: | - -- PREPARE insert_into_WRITING_DELTAS (username, document, ty, si, ei, ipi, s, ft) AS - -- INSERT INTO WRITING_DELTAS - -- (ty, si, ei, ipi, s, ft) - -- VALUES - -- ($1, $2, $3, $4, $5, $6); +stored_procedures: + insert_writing_delta: | + -- PREPARE insert_writing_delta (text, char(48), char(2), integer, integer, integer, text, text) AS + SELECT insert_writing_delta($1, $2, $3, $4, $5, $6, $7, $8); -fetch_writing_deltas : | - PREPARE fetch_writing_deltas (gusername, gdocument) AS - SELECT - WRITING_DELTAS.idx, WRITING_DELTAS.date_created, ty, si, ei, ibi, s, ft - FROM - WRITING_DELTAS, USERS, DOCUMENTS - WHERE - WRITING_DELTAS.user_id = USERS.idx AND - WRITING_DELTAS.document = DOCUMENTS.idx AND - DOCUMENTS.docstring = gdocstring AND - USERS.username = gusername; + fetch_writing_deltas: | + -- PREPARE fetch_writing_deltas (text, char(48)) AS -- username, document string + SELECT + WRITING_DELTAS.idx, WRITING_DELTAS.date_created, ty, si, ei, ibi, s, ft + FROM + WRITING_DELTAS, USERS, DOCUMENTS + WHERE + WRITING_DELTAS.user_id = USERS.idx AND + WRITING_DELTAS.document = DOCUMENTS.idx AND + DOCUMENTS.docstring = $2 AND + USERS.username = $1 + ORDER BY + WRITING_DELTAS.idx; diff --git a/webapp/orm.py b/webapp/orm.py new file mode 100644 index 000000000..fe11ab37b --- /dev/null +++ b/webapp/orm.py @@ -0,0 +1,47 @@ +''' Abstraction to access database ''' +import asyncio +import functools + +import yaml + +import asyncpg + +sql_statements = yaml.safe_load(open("init.sql")) + +conn = None + +stored_procedures = {} + +async def initialize(): + global conn + print("Connecting to database...") + # Connect to the database + conn = await asyncpg.connect() + # Set up tables and stored procedures, if they don't exist. + await conn.execute(sql_statements['init']) + + # Set up stored procedures + for stored_procedure in sql_statements['stored_procedures']: + stored_procedures[stored_procedure] = \ + await conn.prepare(sql_statements['stored_procedures'][stored_procedure]) + print("Connected...") + +asyncio.get_event_loop().run_until_complete(initialize()) + +# TODO: This should be done with a decorator, rather than cut-and-paste +def fetch_writing_deltas(username, docstring): + return stored_procedures['fetch_writing_deltas'].cursor(username, docstring) + +async def insert_writing_delta (username, docstring, ty, si, ei, ibi, s, ft): + rv = await stored_procedures['insert_writing_delta'].fetchval(username, docstring, ty, si, ei, ibi, s, ft) + return rv + +if __name__ == '__main__': + async def test(): + print(await insert_writing_delta ("pmitros", "doc", "ts", 5, 7, 2, "hel", "lo")) + async with conn.transaction(): + cursor = fetch_writing_deltas("pmitros", "doc") + async for record in cursor: + print (record) + + asyncio.get_event_loop().run_until_complete(test()) diff --git a/webapp/webapp.py b/webapp/webapp.py index b68e3f59d..1c2a9a1ee 100644 --- a/webapp/webapp.py +++ b/webapp/webapp.py @@ -4,12 +4,7 @@ import asyncio import asyncpg -def initialize_database(): - conn=asyncpg.connect() - await conn.execute(open("init.sql").read()) - -def add_record(idx, ty, si, ei, ibi, s, ft): - pass +import orm class MainHandler(tornado.web.RequestHandler): def set_default_headers(self): @@ -41,7 +36,7 @@ def make_app(): ]) if __name__ == "__main__": - conn = initialize_database() app = make_app() app.listen(8888) + print("Starting event loop") tornado.ioloop.IOLoop.current().start() From 0b1a048b557c5003ca207332ca4ef9cc2766363d Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sat, 5 Oct 2019 02:23:13 +0100 Subject: [PATCH 010/750] Minimal aiohttp app --- requirements.txt | 4 +- webapp/aio_webapp.py | 48 ++++++++++++++++++++ webapp/init.sql | 58 +++++++++++-------------- webapp/orm.py | 25 ++++++++--- webapp/{webapp.py => torando_webapp.py} | 12 ++++- 5 files changed, 105 insertions(+), 42 deletions(-) create mode 100644 webapp/aio_webapp.py rename webapp/{webapp.py => torando_webapp.py} (82%) diff --git a/requirements.txt b/requirements.txt index b1fd9edc3..51a22c80c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,6 @@ pandas tornado ipython asyncpg -yaml +pyyaml +aiohttp +aiohttp_cors diff --git a/webapp/aio_webapp.py b/webapp/aio_webapp.py new file mode 100644 index 000000000..5034851c5 --- /dev/null +++ b/webapp/aio_webapp.py @@ -0,0 +1,48 @@ +import time + +import asyncio +from aiohttp import web +import aiohttp_cors +from aiohttp.web import middleware + +async def debug_signal(request, handler): + print(request) + +routes = web.RouteTableDef() + +@routes.get('/') +async def hello(request): + print("Request made!") + server_data = { + 'time': time.time(), + 'origin': request.headers.get('Origin', ''), + 'agent': request.headers.get('User-Agent', ''), + 'ip': request.headers.get('X-Real-IP', '') + } + client_data = await request.json() + event = { + 'server': server_data, + 'client': client_data + } + print(event) + print(server_data) + return web.Response(text="Acknowledged!") + +app = web.Application() + +app.on_response_prepare.append(debug_signal) + +app.add_routes([ + web.get('/webapi/', hello), + web.post('/webapi/', hello), +]) + +cors = aiohttp_cors.setup(app, defaults={ + "*": aiohttp_cors.ResourceOptions( + allow_credentials=True, + expose_headers="*", + allow_headers="*", + ) +}) + +web.run_app(app, port=8888) diff --git a/webapp/init.sql b/webapp/init.sql index 6b0749293..016f4be54 100644 --- a/webapp/init.sql +++ b/webapp/init.sql @@ -1,8 +1,8 @@ reset: | DROP TABLE IF EXISTS USERS; - DROP TABLE IF EXISTS WRITING_DELTAS; + DROP TABLE IF EXISTS WRITING_EVENTS; DROP TABLE IF EXISTS DOCUMENTS; - DROP FUNCTION IF EXISTS insert_writing_delta; + DROP FUNCTION IF EXISTS insert_event; init: | CREATE TABLE IF NOT EXISTS USERS ( @@ -18,28 +18,20 @@ init: | date_created timestamp NOT NULL DEFAULT NOW() ); - CREATE TABLE IF NOT EXISTS WRITING_DELTAS ( - idx SERIAL PRIMARY KEY, - user_id integer REFERENCES USERS (idx), -- Who is editing? - document integer REFERENCES DOCUMENTS (idx), -- Which document? - date_created timestamp NOT NULL DEFAULT NOW(), - ty char(2), -- Type of edit. Insert, delete, alter - si integer, -- Start index for deletion - ei integer, -- End index for deletion - ibi integer, -- Insert index - s text, -- Text to insert - ft text -- For debugging: Ongoing reconstruction of full text + CREATE TABLE IF NOT EXISTS WRITING_EVENTS ( + idx SERIAL PRIMARY KEY, + user_id integer REFERENCES USERS (idx), -- Who is editing? + document integer REFERENCES DOCUMENTS (idx), -- Which document? + date_created timestamp NOT NULL DEFAULT NOW(), + event json, + ft text -- For debugging: Ongoing reconstruction of full text ); - CREATE OR REPLACE FUNCTION insert_writing_delta( - gusername text, + CREATE OR REPLACE FUNCTION insert_event( + gusername text, gdocstring char(48), - ty char(2), -- Type of edit. Insert, delete, alter - si integer, -- Start index for deletion - ei integer, -- End index for deletion - ibi integer, -- Insert index - s text, -- Text to insert - ft text -- For debugging: Ongoing reconstruction of full text + event json, + ft text -- For debugging: Ongoing reconstruction of full text ) RETURNS text LANGUAGE plpgsql AS $$ @@ -59,12 +51,12 @@ init: | strresult := strresult || '[New Document]'; INSERT INTO DOCUMENTS (docstring) VALUES (gdocstring); END IF; - -- Insert the writing delta into the database + -- Insert the event into the database with INSERT_ROW_COUNT as - (INSERT INTO WRITING_DELTAS - (user_id, document, ty, si, ei, ibi, s, ft) + (INSERT INTO WRITING_EVENTS + (user_id, document, event) (SELECT - users.idx, documents.idx, ty, si, ei, ibi, s, ft + users.idx, documents.idx, event FROM users, documents where users.username=gusername and documents.docstring=gdocstring) RETURNING 1) @@ -81,20 +73,20 @@ init: | -- Example: SELECT insert_writing_delta('pmitros', 'random-google-doc-id', 'is', 7,8,4,'hello','temp'); stored_procedures: - insert_writing_delta: | + insert_event: | -- PREPARE insert_writing_delta (text, char(48), char(2), integer, integer, integer, text, text) AS - SELECT insert_writing_delta($1, $2, $3, $4, $5, $6, $7, $8); + SELECT insert_event($1, $2, $3, ''); - fetch_writing_deltas: | + fetch_events: | -- PREPARE fetch_writing_deltas (text, char(48)) AS -- username, document string SELECT - WRITING_DELTAS.idx, WRITING_DELTAS.date_created, ty, si, ei, ibi, s, ft + WRITING_EVENTS.idx, WRITING_EVENTS.date_created, event FROM - WRITING_DELTAS, USERS, DOCUMENTS + WRITING_EVENTS, USERS, DOCUMENTS WHERE - WRITING_DELTAS.user_id = USERS.idx AND - WRITING_DELTAS.document = DOCUMENTS.idx AND + WRITING_EVENTS.user_id = USERS.idx AND + WRITING_EVENTS.document = DOCUMENTS.idx AND DOCUMENTS.docstring = $2 AND USERS.username = $1 ORDER BY - WRITING_DELTAS.idx; + WRITING_EVENTS.idx; diff --git a/webapp/orm.py b/webapp/orm.py index fe11ab37b..d7e6395e8 100644 --- a/webapp/orm.py +++ b/webapp/orm.py @@ -2,6 +2,7 @@ import asyncio import functools +import json import yaml import asyncpg @@ -12,11 +13,14 @@ stored_procedures = {} -async def initialize(): +async def initialize(reset=False): global conn print("Connecting to database...") # Connect to the database conn = await asyncpg.connect() + if reset: + await conn.execute(sql_statements['reset']) + # Set up tables and stored procedures, if they don't exist. await conn.execute(sql_statements['init']) @@ -29,18 +33,25 @@ async def initialize(): asyncio.get_event_loop().run_until_complete(initialize()) # TODO: This should be done with a decorator, rather than cut-and-paste -def fetch_writing_deltas(username, docstring): - return stored_procedures['fetch_writing_deltas'].cursor(username, docstring) +def fetch_events(username, docstring): + return stored_procedures['fetch_events'].cursor(username, docstring) -async def insert_writing_delta (username, docstring, ty, si, ei, ibi, s, ft): - rv = await stored_procedures['insert_writing_delta'].fetchval(username, docstring, ty, si, ei, ibi, s, ft) +async def insert_event (username, docstring, event): + rv = await stored_procedures['insert_event'].fetchval(username, docstring, event) return rv if __name__ == '__main__': async def test(): - print(await insert_writing_delta ("pmitros", "doc", "ts", 5, 7, 2, "hel", "lo")) + print(await insert_event ("pmitros", "doc", json.dumps({ + "ty": "ts", + "si": 5, + "ei": 7, + "ibi": 2, + "s": "hel", + "f": "lo" + }))) async with conn.transaction(): - cursor = fetch_writing_deltas("pmitros", "doc") + cursor = fetch_events("pmitros", "doc") async for record in cursor: print (record) diff --git a/webapp/webapp.py b/webapp/torando_webapp.py similarity index 82% rename from webapp/webapp.py rename to webapp/torando_webapp.py index 1c2a9a1ee..1bad228d6 100644 --- a/webapp/webapp.py +++ b/webapp/torando_webapp.py @@ -1,3 +1,10 @@ +''' +Prototyping which framework makes most sense for this application. This may be throw-away code. +''' + + +import json + import tornado.ioloop import tornado.web @@ -17,12 +24,15 @@ def get(self, url): self.set_header("Content-Type", "text/plain") self.write("okay") print(url) + print("GET") print(self.request.body) def post(self, url): self.write("okay") print(url) - print(self.request.body) + print("POST") + event = json.loads(self.request.body) + print(event) def options(self): # no body From eaa19812ddee9f225281daaf247b2d23539a7e60 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 6 Oct 2019 16:36:43 +0100 Subject: [PATCH 011/750] Syncing computers --- webapp/aio_webapp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/webapp/aio_webapp.py b/webapp/aio_webapp.py index 5034851c5..f50fd59bc 100644 --- a/webapp/aio_webapp.py +++ b/webapp/aio_webapp.py @@ -5,6 +5,8 @@ import aiohttp_cors from aiohttp.web import middleware +import orm + async def debug_signal(request, handler): print(request) @@ -24,6 +26,7 @@ async def hello(request): 'server': server_data, 'client': client_data } + # response = await orm.insert_event (username, docstring, event): print(event) print(server_data) return web.Response(text="Acknowledged!") From 8cf3cf16692335e8cbf2b8f65ed866a3d022d881 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 6 Oct 2019 11:37:21 -0400 Subject: [PATCH 012/750] Syncing between computers; moving to unified queue --- extension/background.js | 41 +++++++++++++++++++++-------------------- extension/writing.js | 13 +++++++++---- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/extension/background.js b/extension/background.js index ce278d3d2..2c4a4584b 100644 --- a/extension/background.js +++ b/extension/background.js @@ -14,23 +14,6 @@ function this_a_google_docs_save(request) { return false; } - -function writing_eventlistener(event) { - /* - Here, we process keystroke events and send them to the server. We want fine-grained writing data - with timing. - */ - var event_data = {}; - event_data["event_type"] = "keypress"; - properties = ['altKey', 'charCode', 'code', 'ctrlKey', 'isComposing', 'key', 'keyCode', 'location', 'metaKey', 'repeat', 'shiftKey', 'which', 'isTrusted', 'timeStemp', 'type']; - for (var property in properties) { - event_data[properties[property]] = event[properties[property]]; - } - event_data['date'] = new Date().toLocaleString('en-US'); - console.log(JSON.stringify(event_data)); - writingjs_ajax(event_data); -} - var RAW_DEBUG = false; // Do not save debug requests. We flip this frequently. Perhaps this should be a cookie or browser.storage? chrome.webRequest.onBeforeRequest.addListener( @@ -60,6 +43,7 @@ chrome.webRequest.onBeforeRequest.addListener( from there, being able to easily ignore these is nice. */ function(request) { + chrome.extension.getBackgroundPage().console.log("Web request"); var formdata = {}; if(request.requestBody) { formdata = request.requestBody.formData; @@ -74,20 +58,35 @@ chrome.webRequest.onBeforeRequest.addListener( 'form_data': formdata }); } - if(this_a_google_docs_save(request)) { - writingjs_ajax({ + if(this_a_google_docs_save(request)){ + event = { 'event_type': 'google_docs_save', 'doc_id': googledocs_id_from_url(request.url), 'bundles': JSON.parse(formdata.bundles), 'rev': formdata.rev, 'timestamp': parseInt(request.timeStamp, 10) - }); + }; + chrome.extension.getBackgroundPage().console.log("Google Docs keypress"); + chrome.extension.getBackgroundPage().console.log(event); + writingjs_ajax(event); + } else { + chrome.extension.getBackgroundPage().console.log("Not a save"); } + }, { urls: ["*://docs.google.com/*"/*, "*://mail.google.com/*"*/] }, ['requestBody'] ) +chrome.runtime.onMessage.addListener( + function(request, sender, sendResponse) { + chrome.extension.getBackgroundPage().console.log("Got message"); + chrome.extension.getBackgroundPage().console.log(request); + writingjs_ajax(request); + } +); + + /*chrome.tabs.executeScript({ code: 'console.log("addd")' });*/ @@ -97,3 +96,5 @@ writingjs_ajax({"Loaded now": true}); chrome.identity.getProfileUserInfo(function callback(userInfo) { writingjs_ajax(userInfo); }); + +chrome.extension.getBackgroundPage().console.log("Loaded"); diff --git a/extension/writing.js b/extension/writing.js index 0c1233ba6..62ee8dbd1 100644 --- a/extension/writing.js +++ b/extension/writing.js @@ -25,6 +25,11 @@ function this_is_a_google_doc() { return window.location.href.search("://docs.google.com/") != -1; } +function log_event(event) { + chrome.runtime.sendMessage(event); + //writingjs_ajax(event); +} + function writing_eventlistener(event) { var event_data = {}; event_data["event_type"] = "keypress"; @@ -33,11 +38,11 @@ function writing_eventlistener(event) { event_data[properties[property]] = event[properties[property]]; } event_data['date'] = new Date().toLocaleString('en-US'); - event_data['id'] = doc_id(); + event_data['doc_id'] = doc_id(); console.log(event_data['url']); console.log(JSON.stringify(event_data)); - writingjs_ajax(event_data); + log_event(event_data); } @@ -158,12 +163,12 @@ function google_docs_version_history() { function writing_onload() { if(this_is_a_google_doc()) { - writingjs_ajax({ + log_event({ "event_type": "Google Docs loaded", "partial_text": google_docs_partial_text(), // "partial_html": google_docs_partial_html(), "title": google_docs_title(), - "id": doc_id + "doc_id": doc_id }) google_docs_version_history(); } From 7d7a1de2e71f95559cff5add3c1da49842e23791 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Fri, 11 Oct 2019 19:38:15 +0100 Subject: [PATCH 013/750] Web sockets! --- webapp/aio_webapp.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/webapp/aio_webapp.py b/webapp/aio_webapp.py index f50fd59bc..52b71d1eb 100644 --- a/webapp/aio_webapp.py +++ b/webapp/aio_webapp.py @@ -1,6 +1,7 @@ import time import asyncio +import aiohttp from aiohttp import web import aiohttp_cors from aiohttp.web import middleware @@ -31,10 +32,31 @@ async def hello(request): print(server_data) return web.Response(text="Acknowledged!") +async def websocket_handler(request): + ws = web.WebSocketResponse() + await ws.prepare(request) + + async for msg in ws: + if msg.type == aiohttp.WSMsgType.TEXT: + print(msg) + if msg.data == 'close': + await ws.close() + else: + await ws.send_str(msg.data + '/answer') + elif msg.type == aiohttp.WSMsgType.ERROR: + print('ws connection closed with exception %s' % + ws.exception()) + + print('websocket connection closed') + + return ws + app = web.Application() app.on_response_prepare.append(debug_signal) +app.add_routes([web.get('/wsapi/', websocket_handler)]) + app.add_routes([ web.get('/webapi/', hello), web.post('/webapi/', hello), From ed6b84c14283d4c07a3cc91cb58f5e75e72c80bc Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Fri, 11 Oct 2019 14:43:14 -0400 Subject: [PATCH 014/750] Web Sockets client --- extension/background.js | 64 +++++++++++++++++++++++++++++++++++------ extension/writing.js | 58 +++++++++++++++++-------------------- 2 files changed, 81 insertions(+), 41 deletions(-) diff --git a/extension/background.js b/extension/background.js index 2c4a4584b..46a3d9eef 100644 --- a/extension/background.js +++ b/extension/background.js @@ -2,6 +2,52 @@ Background script. This works across all of Google Chrome. */ +var event_queue = []; +var webSocket = null; + +var writing_lasthash = ""; +function unique_id() { + /* + This function is used to generate a (hopefully) unique ID for + each event. + */ + var shaObj = new jsSHA("SHA-256", "TEXT"); + shaObj.update(writing_lasthash); + shaObj.update(Math.random().toString()); + shaObj.update(Date.now().toString()); + shaObj.update(document.cookie); + shaObj.update("NaCl"); + shaObj.update(window.location.href); + writing_lasthash = shaObj.getHash("HEX"); + return writing_lasthash; +} + +function dequeue_events() { + while(event_queue.length > 0) { + if((webSocket == null) || (webSocket.readyState != 1) ) { + window.setTimeout(reset_websocket, 1000); + return; + } + var event = event_queue.shift(); + webSocket.send(JSON.stringify(event)); + } +} + +function reset_websocket() { + if((webSocket == null) || (webSocket.readyState != 1) ) { + webSocket = new WebSocket("wss://test.mitros.org/wsapi/"); + webSocket.onopen = dequeue_events; + } +} + +function enqueue_event(event) { + event_queue.push(event); + dequeue_events(); +} + +enqueue_event({"event": "extension_loaded"}); +reset_websocket(); + function this_a_google_docs_save(request) { /* Check if this is a Google Docs save request. Return true for something like: @@ -43,7 +89,7 @@ chrome.webRequest.onBeforeRequest.addListener( from there, being able to easily ignore these is nice. */ function(request) { - chrome.extension.getBackgroundPage().console.log("Web request"); + chrome.extension.getBackgroundPage().console.log("Web request:"+request.url); var formdata = {}; if(request.requestBody) { formdata = request.requestBody.formData; @@ -52,13 +98,15 @@ chrome.webRequest.onBeforeRequest.addListener( formdata = {}; } if(RAW_DEBUG) { - writingjs_ajax({ + enqueue_event({ 'event_type': 'raw_request', 'url': request.url, 'form_data': formdata }); } if(this_a_google_docs_save(request)){ + chrome.extension.getBackgroundPage().console.log("Google Docs bundles "+request.url); + console.log(formdata.bundles); event = { 'event_type': 'google_docs_save', 'doc_id': googledocs_id_from_url(request.url), @@ -66,13 +114,11 @@ chrome.webRequest.onBeforeRequest.addListener( 'rev': formdata.rev, 'timestamp': parseInt(request.timeStamp, 10) }; - chrome.extension.getBackgroundPage().console.log("Google Docs keypress"); chrome.extension.getBackgroundPage().console.log(event); - writingjs_ajax(event); + enqueue_event(event); } else { - chrome.extension.getBackgroundPage().console.log("Not a save"); + chrome.extension.getBackgroundPage().console.log("Not a save: "+request.url); } - }, { urls: ["*://docs.google.com/*"/*, "*://mail.google.com/*"*/] }, ['requestBody'] @@ -82,7 +128,7 @@ chrome.runtime.onMessage.addListener( function(request, sender, sendResponse) { chrome.extension.getBackgroundPage().console.log("Got message"); chrome.extension.getBackgroundPage().console.log(request); - writingjs_ajax(request); + enqueue_event(request); } ); @@ -91,10 +137,10 @@ chrome.runtime.onMessage.addListener( code: 'console.log("addd")' });*/ -writingjs_ajax({"Loaded now": true}); +enqueue_event({"Loaded now": true}); chrome.identity.getProfileUserInfo(function callback(userInfo) { - writingjs_ajax(userInfo); + enqueue_event(userInfo); }); chrome.extension.getBackgroundPage().console.log("Loaded"); diff --git a/extension/writing.js b/extension/writing.js index 62ee8dbd1..8bfcb1b43 100644 --- a/extension/writing.js +++ b/extension/writing.js @@ -2,47 +2,46 @@ Page script. This is injected into each web page on associated web sites. */ +/* For debugging purposes: we know the extension is active */ document.body.style.border = "5px solid blue"; -var writing_lasthash = ""; -function unique_id() { - var shaObj = new jsSHA("SHA-256", "TEXT"); - shaObj.update(writing_lasthash); - shaObj.update(Math.random().toString()); - shaObj.update(Date.now().toString()); - shaObj.update(document.cookie); - shaObj.update("NaCl"); - shaObj.update(window.location.href); - writing_lasthash = shaObj.getHash("HEX"); - return writing_lasthash; -} - function doc_id() { + /* + Extract the Google document ID from the window + */ return googledocs_id_from_url(window.location.href); } function this_is_a_google_doc() { + /* + Returns 'true' if we are in a Google Doc + */ return window.location.href.search("://docs.google.com/") != -1; } -function log_event(event) { +function log_event(event_type, event) { + /* + We pass an event, annotated with the page document ID and title, + to the background script + */ + event["title"] = google_docs_title(); + event["doc_id"] = doc_id(); + event['date'] = new Date().toLocaleString('en-US'); + chrome.runtime.sendMessage(event); - //writingjs_ajax(event); } function writing_eventlistener(event) { + /* + Listen for keystroke events, and pass them back to the background page. + */ var event_data = {}; event_data["event_type"] = "keypress"; properties = ['altKey', 'charCode', 'code', 'ctrlKey', 'isComposing', 'key', 'keyCode', 'location', 'metaKey', 'repeat', 'shiftKey', 'which', 'isTrusted', 'timeStemp', 'type']; for (var property in properties) { event_data[properties[property]] = event[properties[property]]; } - event_data['date'] = new Date().toLocaleString('en-US'); - event_data['doc_id'] = doc_id(); - console.log(event_data['url']); - - console.log(JSON.stringify(event_data)); - log_event(event_data); + log_event("keystroke", event_data); } @@ -60,7 +59,6 @@ for(iframe in iframes){ } } - function gmail_text() { /* This function returns all the editable text in the current gmail @@ -142,10 +140,12 @@ function google_docs_version_history() { var first_revision = tiles.firstRev; var last_revision = tiles.tileInfo[tiles.tileInfo.length - 1].end; version_history_url = "https://docs.google.com/document/d/"+doc_id()+"/revisions/load?id="+doc_id()+"&start="+first_revision+"&end="+last_revision; - console.log(version_history_url); fetch(version_history_url).then(function(history_response) { history_response.text().then(function(history_text) { - console.log(history_text); + log_event( + "document_history", + {'history': history_text} + ) }); }); }); @@ -163,17 +163,11 @@ function google_docs_version_history() { function writing_onload() { if(this_is_a_google_doc()) { - log_event({ - "event_type": "Google Docs loaded", - "partial_text": google_docs_partial_text(), - // "partial_html": google_docs_partial_html(), - "title": google_docs_title(), - "doc_id": doc_id + log_event("document_loaded", { + "partial_text": google_docs_partial_text() }) google_docs_version_history(); } } window.addEventListener("load", writing_onload); - -console.log(unique_id()); From d5de5c11eb5bee476da5bbfc64fcba0da7ffdc98 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Thu, 16 Jan 2020 09:26:51 -0500 Subject: [PATCH 015/750] Syncing with Oren --- extension/writing.js | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/extension/writing.js b/extension/writing.js index 8bfcb1b43..672e91b33 100644 --- a/extension/writing.js +++ b/extension/writing.js @@ -5,6 +5,14 @@ Page script. This is injected into each web page on associated web sites. /* For debugging purposes: we know the extension is active */ document.body.style.border = "5px solid blue"; +/* On startup, log the identity information the browser has. We want oauth at some point, but + perhaps not at all points. */ +chrome.identity.getProfileInfo(function(userInfo)) { + log_event("chrome_identity", {"email": userInfo.email, + "id": userInfo.id + }); +} + function doc_id() { /* Extract the Google document ID from the window @@ -144,23 +152,14 @@ function google_docs_version_history() { history_response.text().then(function(history_text) { log_event( "document_history", - {'history': history_text} - ) + {'history': JSON.parse(history_text.substring(4))} + ); }); }); }); }); } -// https://docs.google.com/document/d/1lt_lSfEM9jd7Ga6uzENS_s8ZajcxpE0cKuzXbDoBoyU/revisions/load?id=1lt_lSfEM9jd7Ga6uzENS_s8ZajcxpE0cKuzXbDoBoyU&start=1&end=300 - -// {"tileInfo":[{"start":1,"end":1,"endMillis":1565618108628,"users":["10308288613201963581"],"systemRevs":[],"expandable":false,"revisionMac":"DXjr1gUIKJye-Q"},{"start":2,"end":25,"endMillis":1565618606428,"users":["10308288613201963581"],"systemRevs":[],"expandable":true,"revisionMac":"tzzZJD5qs1k8YQ"},{"start":26,"end":26,"endMillis":1565687165389,"users":["10308288613201963581"],"systemRevs":[],"expandable":false,"revisionMac":"EFaHxMq5RwIbiA"},{"start":27,"end":131,"endMillis":1565887557347,"users":["10308288613201963581"],"systemRevs":[],"expandable":true,"revisionMac":"IktkTE-OHb_a4w"},{"start":132,"end":199,"endMillis":1565895985511,"users":["10308288613201963581"],"systemRevs":[],"expandable":true,"revisionMac":"4-jj6h3QR26WOA"},{"start":200,"end":238,"endMillis":1565906888993,"users":["10308288613201963581"],"systemRevs":[],"expandable":true,"revisionMac":"u3yYJvJBiVfImg"},{"start":239,"end":243,"endMillis":1566036118386,"users":["10308288613201963581"],"systemRevs":[],"expandable":false,"revisionMac":"fGom_rH071Lfzg"},{"start":244,"end":248,"endMillis":1566692983735,"users":["10308288613201963581"],"systemRevs":[],"expandable":false,"revisionMac":"sxMGFwhfT9Akbg"},{"start":249,"end":262,"endMillis":1566748493846,"users":["10308288613201963581"],"systemRevs":[],"expandable":true,"revisionMac":"Rya8bp1glL8XtQ"},{"start":263,"end":266,"endMillis":1566814345191,"users":["10308288613201963581"],"systemRevs":[],"expandable":false,"revisionMac":"K7B8GTV1UvSR_g"},{"start":267,"end":268,"endMillis":1566830127167,"users":["10308288613201963581"],"systemRevs":[],"expandable":false,"revisionMac":"jEfJ5MRCmwhZlg"},{"start":269,"end":301,"endMillis":1566838256185,"users":["10308288613201963581"],"systemRevs":[],"expandable":true,"revisionMac":"uy2uqc9q0pBQnQ"},{"start":302,"end":302,"endMillis":1566840996445,"users":["10308288613201963581"],"systemRevs":[],"expandable":false,"revisionMac":"pAikpOJ2fomhZg"},{"start":303,"end":307,"endMillis":1566849461292,"users":["10308288613201963581"],"systemRevs":[],"expandable":true,"revisionMac":"OR8lknfQhBqmuQ"}],"userMap":{"10308288613201963581":{"name":"Piotr Mitros","photo":"//lh3.googleusercontent.com/a-/AAuE7mBPq-BL5P1ZZg1N0mh9BPLSSkgBqeN3wnLVH0tZ\u003ds50-c-k-no","defaultPhoto":false,"color":"#26A69A","anonymous":false}},"firstRev":1} - -// cookie: S=documents=-0139lI8XPtj5RlM3yMiSMm65NpEPhb1; SID=nAd7sht2a_QMpTbmZR0YK3gCseKEBX_ie8HEMZIDsv9btLkzJuNKN15D4IzbSWAL8eym7Q.; HSID=APez4e6AmTL1EVx2o; SSID=Az1RQ_epFl_39Hmwk; APISID=DY4P1zvmTBZhHgP5/A1yeGjd8U74GTzJ-x; SAPISID=yGBiRuDtlmqqwzs4/AsINvELYvon8jWvGt; ANID=AHWqTUk__nHk8xzlNlnhj60_p6JhVxs6Q7-kDkQvaG82i-nU7_PG1q2K0i96y5S9; SEARCH_SAMESITE=CgQI1o0B; S=explorer=PDpeyuS6wzjoz7SlIiZ9Y6dAiLQ0AGKP; NID=188=ZppA1vlmd9N7uDI7Re0NOBuGAAi9Fgv5CYEMw1y0akPcMuSHTIBmd_MWivVercGRr-ZCOIadtyhK8Rm9ZUzcqwAzJOeXGnALMqpuPamh0lQguPPZjjldOckFCaPSrICJUCZ4zW31hAyGNBtPMFe-SYe6UdIBN_k5DlsprTZDUc_fmFpMbgEwlGG_kIKyonhpWX4OdkaUYPLJJCYqCNrJSEA3YpLkTFyY5ArVd2HthIyd7UpshgxAYw1lqh_XHIEe1Kb4v2rSeS9IuOl70CoOqydi69b6N5bjoGc2qi_QZeFmSLp-RU72XYpcGVB5dNVC-mvUsNzfRQaUyaonl7I44_R6Off6aZ3TwiOe5ZnPxw2BiqdPqsE3CgeZi4ZdSw3JXXPliNMlvt9IIwuQ7aDzaGbLH-CFueO6g8B92JDGcJ0P; 1P_JAR=2019-8-26-19; SIDCC=AN0-TYvNZxiSjuj5_gFXAjiyFnr1RDCQjEedbdU0SHElyA1F12Cek9tYKjiDrDJhOdX7YvQb_lfPVg; GFE_RTT=71 - -// } - - function writing_onload() { if(this_is_a_google_doc()) { log_event("document_loaded", { From 5808208c4e4a8c99402a31d37c9d5e2ce9cfda6b Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Thu, 16 Jan 2020 10:01:19 -0500 Subject: [PATCH 016/750] Ansible configuration prototype --- configuration/local.yaml | 6 +++++ configuration/tasks/writing.yaml | 44 ++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 configuration/local.yaml create mode 100644 configuration/tasks/writing.yaml diff --git a/configuration/local.yaml b/configuration/local.yaml new file mode 100644 index 000000000..d12102f87 --- /dev/null +++ b/configuration/local.yaml @@ -0,0 +1,6 @@ +- name: Provision writing analysis server + hosts: localhost + connection: local + tasks: + - include: tasks/writing.yaml + diff --git a/configuration/tasks/writing.yaml b/configuration/tasks/writing.yaml new file mode 100644 index 000000000..dfc2e63e0 --- /dev/null +++ b/configuration/tasks/writing.yaml @@ -0,0 +1,44 @@ +- apt: upgrade=dist update_cache=yes + +- name: Basic utils + apt: name={{ item }} + with_items: + - curl + - emacs + - git + - git-core + - git-core + - links + - lynx + - mosh + - nmap + - whois + - screen + - wipe + - build-essential + - awscli + +- name: Python + apt: name={{ item }} + with_items: + - ipython3 + - libxml2-dev + - libxslt1-dev + - python3-boto + - python3-bson + - python3-dev + - python3-matplotlib + - python3-numpy + - python3-pandas + - python3-pip + - python3-scipy + - python3-setuptools + - python3-sklearn + - virtualenvwrapper + - libjpeg-dev + - python3-opencv + - python3-virtualenv + +- name: Server + - postgresql + - nginx \ No newline at end of file From d98c4d25bc4e32c4ee0000f7a8638644a8490985 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Thu, 16 Jan 2020 10:04:25 -0500 Subject: [PATCH 017/750] bug --- configuration/tasks/writing.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configuration/tasks/writing.yaml b/configuration/tasks/writing.yaml index dfc2e63e0..78e333a7f 100644 --- a/configuration/tasks/writing.yaml +++ b/configuration/tasks/writing.yaml @@ -40,5 +40,7 @@ - python3-virtualenv - name: Server + apt: name={{ item }} + with_items: - postgresql - nginx \ No newline at end of file From 68cf82a77e42b76907441b7a4414a22e84c65d8a Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Thu, 16 Jan 2020 10:11:33 -0500 Subject: [PATCH 018/750] Removed redunant line --- configuration/tasks/writing.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/configuration/tasks/writing.yaml b/configuration/tasks/writing.yaml index 78e333a7f..8a5794d2e 100644 --- a/configuration/tasks/writing.yaml +++ b/configuration/tasks/writing.yaml @@ -7,7 +7,6 @@ - emacs - git - git-core - - git-core - links - lynx - mosh From da69c14900cb08c72244d503b67154a40e699d71 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Thu, 16 Jan 2020 11:40:00 -0500 Subject: [PATCH 019/750] certbot --- configuration/tasks/writing.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configuration/tasks/writing.yaml b/configuration/tasks/writing.yaml index 8a5794d2e..78d211a77 100644 --- a/configuration/tasks/writing.yaml +++ b/configuration/tasks/writing.yaml @@ -42,4 +42,5 @@ apt: name={{ item }} with_items: - postgresql - - nginx \ No newline at end of file + - nginx + - certbot From d63a88efc570ce885a7b449d7308d2e03e75fb28 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Thu, 16 Jan 2020 11:50:20 -0500 Subject: [PATCH 020/750] Attempt to add nginx locations. Syncing to test if it works --- configuration/files/nginx-locations | 30 +++++++++++++ configuration/scripts/add_nginx_locations.py | 45 ++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 configuration/files/nginx-locations create mode 100644 configuration/scripts/add_nginx_locations.py diff --git a/configuration/files/nginx-locations b/configuration/files/nginx-locations new file mode 100644 index 000000000..3bb4a0a75 --- /dev/null +++ b/configuration/files/nginx-locations @@ -0,0 +1,30 @@ + + location /webapi/ { + proxy_pass http://localhost:8888/webapi/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + if ($request_method = OPTIONS ) { + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + return 200; + } + } + + location /wsapi/ { + proxy_pass http://localhost:8888/wsapi/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400; + + if ($request_method = OPTIONS ) { + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + return 200; + } + } + diff --git a/configuration/scripts/add_nginx_locations.py b/configuration/scripts/add_nginx_locations.py new file mode 100644 index 000000000..f695f9396 --- /dev/null +++ b/configuration/scripts/add_nginx_locations.py @@ -0,0 +1,45 @@ +""" +This script adds the locations for our web API to nginx. It adds +them after the default location. +""" + +import sys +import shutil +import datetime + +lines = open("/etc/nginx/sites-enabled/default", "r").readlines() + +# If we've already added these, do nothing. +for line in lines: + if "webapi" in line: + print("Already configured!") + sys.exit(-1) + +# We will accumulate the new file into this variable +output = "" + +# We step through the file until we find the first 'location' line, and +# keep cycling until we find a single "}" ending that section. +# +# At that point, we add the new set of location + +location_found = False +added = False +for line in lines: + output += line + if line.strip().startswith("location"): + print("Found") + location_found = True + if location_found and line.strip() == "}" and not added: + output += open("../files/nginx-locations").read() + added = True + + +backup_file = "/etc/nginx/sites-enabled-default-" + \ + datetime.datetime.utcnow().isoformat() +shutil.move("/etc/nginx/sites-enabled/default", backup_file) + +with open("/etc/nginx/sites-enabled/default", "w") as fp: + fp.write(output) + +print(output) From 7428ab5f86d846a7cabb018b08c66300022b07a1 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Thu, 16 Jan 2020 11:58:31 -0500 Subject: [PATCH 021/750] Python server libraries to ansible --- configuration/tasks/writing.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configuration/tasks/writing.yaml b/configuration/tasks/writing.yaml index 78d211a77..c178b50ee 100644 --- a/configuration/tasks/writing.yaml +++ b/configuration/tasks/writing.yaml @@ -37,6 +37,11 @@ - libjpeg-dev - python3-opencv - python3-virtualenv + - python3-aiohttp + - python3-aiohttp-cors + - python3-tornado + - python3-yaml + - python3-asyncpg - name: Server apt: name={{ item }} From 785c8e11a62eef1c68acab0c12adf4500a5f6370 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Thu, 16 Jan 2020 12:44:03 -0500 Subject: [PATCH 022/750] Feature flags for web sockets versus AJAX, commenting out ORM --- extension/background.js | 16 ++++++++++++---- extension/writing_common.js | 6 ++++-- webapp/aio_webapp.py | 2 +- webapp/torando_webapp.py | 2 +- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/extension/background.js b/extension/background.js index 46a3d9eef..95d39c47d 100644 --- a/extension/background.js +++ b/extension/background.js @@ -35,18 +35,26 @@ function dequeue_events() { function reset_websocket() { if((webSocket == null) || (webSocket.readyState != 1) ) { - webSocket = new WebSocket("wss://test.mitros.org/wsapi/"); + webSocket = new WebSocket("wss://writing.hopto.org/wsapi/"); webSocket.onopen = dequeue_events; } } function enqueue_event(event) { - event_queue.push(event); - dequeue_events(); + if(EXPERIMENTAL_WEBSOCKET) { + event_queue.push(event); + dequeue_events(); + } + else { + writingjs_ajax(event) + } } enqueue_event({"event": "extension_loaded"}); -reset_websocket(); + +if(EXPERIMENTAL_WEBSOCKET) { + reset_websocket(); +} function this_a_google_docs_save(request) { /* diff --git a/extension/writing_common.js b/extension/writing_common.js index b7e54d126..855711b36 100644 --- a/extension/writing_common.js +++ b/extension/writing_common.js @@ -1,4 +1,6 @@ -var WRITINGJS_SERVER = "https://test.mitros.org/webapi/"; +var WRITINGJS_AJAX_SERVER = "https://writing.hopto.org/webapi/"; +var WRITINGJS_WSS_SERVER = "https://writing.hopto.org/webapi/"; +var EXPERIMENTAL_WEBSOCKET = false; function writingjs_ajax(data) { /* @@ -12,7 +14,7 @@ function writingjs_ajax(data) { httpRequest = new XMLHttpRequest(); //httpRequest.withCredentials = true; - httpRequest.open("POST", WRITINGJS_SERVER); + httpRequest.open("POST", WRITINGJS_AJAX_SERVER); httpRequest.send(JSON.stringify(data)); } diff --git a/webapp/aio_webapp.py b/webapp/aio_webapp.py index 52b71d1eb..f9bc5c841 100644 --- a/webapp/aio_webapp.py +++ b/webapp/aio_webapp.py @@ -6,7 +6,7 @@ import aiohttp_cors from aiohttp.web import middleware -import orm +# import orm async def debug_signal(request, handler): print(request) diff --git a/webapp/torando_webapp.py b/webapp/torando_webapp.py index 1bad228d6..862762be5 100644 --- a/webapp/torando_webapp.py +++ b/webapp/torando_webapp.py @@ -11,7 +11,7 @@ import asyncio import asyncpg -import orm +# import orm class MainHandler(tornado.web.RequestHandler): def set_default_headers(self): From aeabee63b676dd0df725feaec23a516ae54e7f93 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Thu, 16 Jan 2020 13:09:12 -0500 Subject: [PATCH 023/750] Working nginx configuration in a file --- configuration/files/default | 94 +++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 configuration/files/default diff --git a/configuration/files/default b/configuration/files/default new file mode 100644 index 000000000..e2a5bd428 --- /dev/null +++ b/configuration/files/default @@ -0,0 +1,94 @@ +## +# You should look at the following URL's in order to grasp a solid understanding +# of Nginx configuration files in order to fully unleash the power of Nginx. +# https://www.nginx.com/resources/wiki/start/ +# https://www.nginx.com/resources/wiki/start/topics/tutorials/config_pitfalls/ +# https://wiki.debian.org/Nginx/DirectoryStructure +# +# In most cases, administrators will remove this file from sites-enabled/ and +# leave it as reference inside of sites-available where it will continue to be +# updated by the nginx packaging team. +# +# This file will automatically load configuration files provided by other +# applications, such as Drupal or Wordpress. These applications will be made +# available underneath a path with that package name, such as /drupal8. +# +# Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples. +## + +# Default server configuration +# +server { + listen 80 default_server; + listen [::]:80 default_server; + + server_name [[[[[[SERVERNAME]]]]]; # managed by Certbot + listen [::]:443 ssl ipv6only=on; # managed by Certbot + listen 443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/writing.hopto.org/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/writing.hopto.org/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + + + root /var/www/html; + + # Add index.php to the list if you are using PHP + index index.html index.htm index.nginx-debian.html; + + server_name writing.hopto.org; + + location / { + # First attempt to serve request as file, then + # as directory, then fall back to displaying a 404. + try_files $uri $uri/ =404; + } + + location /webapi/ { + proxy_pass http://localhost:8888/webapi/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + if ($request_method = OPTIONS ) { + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + return 200; + } + } + + location /wsapi/ { + proxy_pass http://localhost:8888/wsapi/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400; + + if ($request_method = OPTIONS ) { + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + return 200; + } + } + + + # pass PHP scripts to FastCGI server + # + #location ~ \.php$ { + # include snippets/fastcgi-php.conf; + # + # # With php-fpm (or other unix sockets): + # fastcgi_pass unix:/var/run/php/php7.0-fpm.sock; + # # With php-cgi (or other tcp sockets): + # fastcgi_pass 127.0.0.1:9000; + #} + + # deny access to .htaccess files, if Apache's document root + # concurs with nginx's one + # + #location ~ /\.ht { + # deny all; + #} +} \ No newline at end of file From 53df44892173d214a25234748b55bacc1071a325 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Thu, 16 Jan 2020 13:13:15 -0500 Subject: [PATCH 024/750] Fixed extension bug -- should work now --- extension/writing.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extension/writing.js b/extension/writing.js index 672e91b33..0904a7965 100644 --- a/extension/writing.js +++ b/extension/writing.js @@ -7,11 +7,11 @@ document.body.style.border = "5px solid blue"; /* On startup, log the identity information the browser has. We want oauth at some point, but perhaps not at all points. */ -chrome.identity.getProfileInfo(function(userInfo)) { +/*chrome.identity.getProfileInfo(function(userInfo) { log_event("chrome_identity", {"email": userInfo.email, "id": userInfo.id }); -} +});*/ function doc_id() { /* From d23c51c29fc2239c99bf0a54691a3714ede9d8c1 Mon Sep 17 00:00:00 2001 From: Oren Livne Date: Tue, 21 Jan 2020 09:47:14 -0500 Subject: [PATCH 025/750] Added informal installing instructions for our project. --- INSTALL.md | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 INSTALL.md diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 000000000..1bef1ef9f --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,73 @@ +# Formative Process for Writing - INstallation Instructions +Last updated: 16-JAN-2020 + +## Install Chrome Extension +* Set up a github SSH key. +* Download the extension code: +```bash +cd +git clone https://github.com/ETS-Next-Gen/writing_analysis.git writing_analysis +``` +* Navigate to `chrome://extensions`. +* Click on "Load Unpacked". Select ~/writing_anlysis/extensions + +## Create an AWS account and an EC2 instance. +* Select Ubuntu, nano AMI. The cost should be 0.5 cents per hour. +* In security groups, add HTTP, HTTPS rules. Open up only to your computer's IP address (the client). +* Launch instance. +* Suppose your instance public DNS is {ec2_ip}, e.g., ec2-18-223-122-172.us-east-2.compute.amazonaws.com. +* Create aSSH key pair, save PEM file say under ~/.ssh, chmod to u+r. + +## Set up the EC2 instance. +* SSH into the machine: `bash ssh -i {pem_file} ubuntu@{ec2_ip}`. +* (Optional) Create a user account: sudo useradd {user} +* Download the server code (same repository as the extension): +```bash +cd +git clone https://github.com/ETS-Next-Gen/writing_analysis.git writing_analysis +``` +* Install Ansible. +```bash +sudo apt-get update +sudo apt-get upgrade +sudo apt-get install ansible +```` +* Configure Ansible. +sudo pico /etc/ansible/hosts +Add +``` +[localhost] +127.0.0.1 +``` +* `cd ~/writing_analysis/configuration`. +* Run `sudo ansible-playbook local.yaml`. This may take a while on an EC2 nanon machine. +If all goes well, you should see an output with no errors, like this: +``` +bash +... +PLAY RECAP ****************************************************************************************** +127.0.0.1 : ok=5 changed=4 unreachable=0 failed=0 +``` +* Navigate to http://{ec2_ip}; you should see the message "Welcome to nginx!" if it's working. + +## Obtain a free domain name +* Go to noip.com. +* Sign up + +## Obtain a free SSL Certificate Using Certbot +* Run the following commands: +```bash +sudo apt-get install software-properties-common +Sudo add-apt-repository universe +sudo add-apt-repository ppa:certbot/certbot +sudo apt-get update +sudo apt-get install certbot python-certbot-nginx +``` +``` +bash +sudo certbot --nginx +``` +-- Put in your {mydomain}.hopto.org address. +-- Choose 1 - no redirect. + +## Stand up a backend server on the EC2 instance. From f165a8bee267a9e5d5b3de2efefa2bb19772cb02 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sat, 8 Feb 2020 20:47:34 -0500 Subject: [PATCH 026/750] First prototype options page --- extension/background.js | 8 +++++ extension/manifest.json | 6 +++- extension/{popup => pages}/action.css | 0 extension/pages/options.html | 18 ++++++++++ extension/pages/options.js | 39 ++++++++++++++++++++ extension/{popup => pages}/settings.html | 2 +- extension/writing.js | 45 ++++++++++++++++-------- extension/writing_common.js | 8 +++++ 8 files changed, 110 insertions(+), 16 deletions(-) rename extension/{popup => pages}/action.css (100%) create mode 100644 extension/pages/options.html create mode 100644 extension/pages/options.js rename extension/{popup => pages}/settings.html (87%) diff --git a/extension/background.js b/extension/background.js index 95d39c47d..4237291d1 100644 --- a/extension/background.js +++ b/extension/background.js @@ -5,6 +5,14 @@ Background script. This works across all of Google Chrome. var event_queue = []; var webSocket = null; +/* On startup, log the identity information the browser has. We want oauth at some point, but + perhaps not at all points. */ +/*chrome.identity.getProfileInfo(function(userInfo) { + log_event("chrome_identity", {"email": userInfo.email, + "id": userInfo.id + }); +});*/ + var writing_lasthash = ""; function unique_id() { /* diff --git a/extension/manifest.json b/extension/manifest.json index 335db4a4d..89e670b43 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -10,7 +10,7 @@ "browser_action": { "default_title": "Writing Process", - "default_popup": "popup/settings.html", + "default_popup": "pages/settings.html", "default_icon": { "48": "icons/lousy-fountain-pen-48.png" } @@ -34,5 +34,9 @@ ], "icons": { "48": "icons/lousy-fountain-pen-48.png" + }, + "options_ui": { + "page": "pages/options.html", + "chrome_style": true } } diff --git a/extension/popup/action.css b/extension/pages/action.css similarity index 100% rename from extension/popup/action.css rename to extension/pages/action.css diff --git a/extension/pages/options.html b/extension/pages/options.html new file mode 100644 index 000000000..0813e6dca --- /dev/null +++ b/extension/pages/options.html @@ -0,0 +1,18 @@ + + + + + + + + +

Server: no value found

+
+ + + +
+ + + + diff --git a/extension/pages/options.js b/extension/pages/options.js new file mode 100644 index 000000000..8dd7206fa --- /dev/null +++ b/extension/pages/options.js @@ -0,0 +1,39 @@ +/* + Documentation on how to create an options page + */ + +function saveServerToStorage(new_server) { + console.log("Saving: "+new_server); + chrome.storage.sync.set({ + "process-server": new_server + }, restoreOptions); +} + +function saveOptions(e) { + /* + Callback when user hits "save" on the options page + */ + var new_server = document.querySelector("#process-server").value; + saveServerToStorage(new_server); + e.preventDefault(); +} + +function restoreOptions() { + /* + Initialize the options page for the extension. Eventually, we'd + like to also use chrome.storage.managed so that school admins + can set these settings up centrally, without student overrides + */ + chrome.storage.sync.get(['process-server'], function(result){ + var sync_storage_server = result['process-server']; + console.log("Loaded saved server: " + sync_storage_server); + if(!sync_storage_server) { + sync_storage_server = "writing.mitros.org"; + } + document.querySelector("#current-process-server").innerText = sync_storage_server; + document.querySelector("#process-server").value = sync_storage_server; + }); +} + +document.addEventListener('DOMContentLoaded', restoreOptions); +document.querySelector("form").addEventListener("submit", saveOptions); diff --git a/extension/popup/settings.html b/extension/pages/settings.html similarity index 87% rename from extension/popup/settings.html rename to extension/pages/settings.html index fda3046f9..381fb76dc 100644 --- a/extension/popup/settings.html +++ b/extension/pages/settings.html @@ -4,7 +4,7 @@ -

Writting Process

+

Writing Process

  • See my stats
    • diff --git a/extension/writing.js b/extension/writing.js index 0904a7965..120600dc3 100644 --- a/extension/writing.js +++ b/extension/writing.js @@ -1,23 +1,28 @@ /* -Page script. This is injected into each web page on associated web sites. + Page script. This is injected into each web page on associated web sites. */ /* For debugging purposes: we know the extension is active */ document.body.style.border = "5px solid blue"; -/* On startup, log the identity information the browser has. We want oauth at some point, but - perhaps not at all points. */ -/*chrome.identity.getProfileInfo(function(userInfo) { - log_event("chrome_identity", {"email": userInfo.email, - "id": userInfo.id - }); -});*/ +function log_error(error_string) { + /* + We should send errors to the server, but for now, we + log to the console. + */ + console.log(error_string); +} function doc_id() { /* Extract the Google document ID from the window - */ - return googledocs_id_from_url(window.location.href); + */ + try { + return googledocs_id_from_url(window.location.href); + } catch(error) { + log_error("Couldn't read document id"); + return null; + } } function this_is_a_google_doc() { @@ -87,9 +92,16 @@ function gmail_text() { function google_docs_title() { /* - Return the title of a Google Docs document - */ - return document.getElementsByClassName("docs-title-input")[0].value; + Return the title of a Google Docs document. + + Note this is not guaranteed 100% reliable. + */ + try { + return document.getElementsByClassName("docs-title-input")[0].value; + } catch(error) { + log_error("Couldn't read document title"); + return null; + } } function google_docs_partial_text() { @@ -99,7 +111,12 @@ function google_docs_partial_text() { pages may be lazy-loaded. The text omits formatting, which is helpful for many types of analysis */ - return document.getElementsByClassName("kix-page")[0].innerText; + try { + return document.getElementsByClassName("kix-page")[0].innerText; + } catch(error) { + log_error("Could get document text"); + return null; + } } function google_docs_partial_html() { diff --git a/extension/writing_common.js b/extension/writing_common.js index 855711b36..1b3a51976 100644 --- a/extension/writing_common.js +++ b/extension/writing_common.js @@ -2,6 +2,14 @@ var WRITINGJS_AJAX_SERVER = "https://writing.hopto.org/webapi/"; var WRITINGJS_WSS_SERVER = "https://writing.hopto.org/webapi/"; var EXPERIMENTAL_WEBSOCKET = false; + +chrome.storage.sync.get(['process-server'], function(result) { + var WRITINGJS_AJAX_SERVER = result['process-server']; + if(!WRITINGJS_AJAX_SERVER) { + WRITINGJS_AJAX_SERVER = "https://writing.hopto.org/webapi/"; + } +}); + function writingjs_ajax(data) { /* Helper function to send a logging AJAX request to the server. From 600d8aedc8c4be7475a4d97f45f8bd45e4843d9b Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sat, 8 Feb 2020 21:01:49 -0500 Subject: [PATCH 027/750] Moving all AJAX into the background page --- extension/background.js | 28 ++++++++++++++++++++++++++++ extension/writing_common.js | 29 ----------------------------- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/extension/background.js b/extension/background.js index 4237291d1..4b746a0c2 100644 --- a/extension/background.js +++ b/extension/background.js @@ -13,6 +13,34 @@ var webSocket = null; }); });*/ +var WRITINGJS_AJAX_SERVER = "https://writing.hopto.org/webapi/"; +var WRITINGJS_WSS_SERVER = "https://writing.hopto.org/webapi/"; +var EXPERIMENTAL_WEBSOCKET = false; + + +chrome.storage.sync.get(['process-server'], function(result) { + var WRITINGJS_AJAX_SERVER = result['process-server']; + if(!WRITINGJS_AJAX_SERVER) { + WRITINGJS_AJAX_SERVER = "https://writing.hopto.org/webapi/"; + } +}); + +function writingjs_ajax(data) { + /* + Helper function to send a logging AJAX request to the server. + This function takes a JSON dictionary of data. + + TODO: Convert to a queue for offline operation using Chrome + Storage API? Cache to Chrome Storage? Chrome Storage doesn't + support meaningful concurrency, + */ + + httpRequest = new XMLHttpRequest(); + //httpRequest.withCredentials = true; + httpRequest.open("POST", WRITINGJS_AJAX_SERVER); + httpRequest.send(JSON.stringify(data)); +} + var writing_lasthash = ""; function unique_id() { /* diff --git a/extension/writing_common.js b/extension/writing_common.js index 1b3a51976..9bd7cccc1 100644 --- a/extension/writing_common.js +++ b/extension/writing_common.js @@ -1,32 +1,3 @@ -var WRITINGJS_AJAX_SERVER = "https://writing.hopto.org/webapi/"; -var WRITINGJS_WSS_SERVER = "https://writing.hopto.org/webapi/"; -var EXPERIMENTAL_WEBSOCKET = false; - - -chrome.storage.sync.get(['process-server'], function(result) { - var WRITINGJS_AJAX_SERVER = result['process-server']; - if(!WRITINGJS_AJAX_SERVER) { - WRITINGJS_AJAX_SERVER = "https://writing.hopto.org/webapi/"; - } -}); - -function writingjs_ajax(data) { - /* - Helper function to send a logging AJAX request to the server. - This function takes a JSON dictionary of data. - - TODO: Convert to a queue for offline operation using Chrome - Storage API? Cache to Chrome Storage? Chrome Storage doesn't - support meaningful concurrency, - */ - - httpRequest = new XMLHttpRequest(); - //httpRequest.withCredentials = true; - httpRequest.open("POST", WRITINGJS_AJAX_SERVER); - httpRequest.send(JSON.stringify(data)); -} - - function googledocs_id_from_url(url) { /* Given a URL like: From 023abac062df7e7ec238b280a8f52180d79d296a Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sat, 8 Feb 2020 23:03:18 -0500 Subject: [PATCH 028/750] Starting a major refactor to use queues --- extension/background.js | 156 ++++++++++++++++++++---------------- extension/writing_common.js | 16 ++++ 2 files changed, 105 insertions(+), 67 deletions(-) diff --git a/extension/background.js b/extension/background.js index 4b746a0c2..88bc2c6a3 100644 --- a/extension/background.js +++ b/extension/background.js @@ -2,28 +2,52 @@ Background script. This works across all of Google Chrome. */ + var event_queue = []; + +/* To avoid race conditions, we keep track of events we've successfully sent */ +var sent_events = new Set(); + var webSocket = null; -/* On startup, log the identity information the browser has. We want oauth at some point, but - perhaps not at all points. */ -/*chrome.identity.getProfileInfo(function(userInfo) { - log_event("chrome_identity", {"email": userInfo.email, - "id": userInfo.id - }); -});*/ +//var WRITINGJS_AJAX_SERVER = "https://writing.hopto.org/webapi/"; +//var WRITINGJS_WSS_SERVER = "https://writing.hopto.org/webapi/"; + +var WRITINGJS_AJAX_SERVER = null; -var WRITINGJS_AJAX_SERVER = "https://writing.hopto.org/webapi/"; -var WRITINGJS_WSS_SERVER = "https://writing.hopto.org/webapi/"; var EXPERIMENTAL_WEBSOCKET = false; +/* + FSM + + +----------------------+ + | Load server settings | + | from chrome.storage | + +----------------------+ + | + v + +-------------------+ + | Connect to server | + +-------------------+ + + Load events queue + from chrome.storage + + +Dequeue events +*/ + +function dequeue_events() { +/* while(event_queue.length > 0) { + if((webSocket == null) || (webSocket.readyState != 1) ) { + window.setTimeout(reset_websocket, 1000); + return; + } + var event = event_queue.shift(); + webSocket.send(JSON.stringify(event)); + }*/ +} -chrome.storage.sync.get(['process-server'], function(result) { - var WRITINGJS_AJAX_SERVER = result['process-server']; - if(!WRITINGJS_AJAX_SERVER) { - WRITINGJS_AJAX_SERVER = "https://writing.hopto.org/webapi/"; - } -}); function writingjs_ajax(data) { /* @@ -41,41 +65,6 @@ function writingjs_ajax(data) { httpRequest.send(JSON.stringify(data)); } -var writing_lasthash = ""; -function unique_id() { - /* - This function is used to generate a (hopefully) unique ID for - each event. - */ - var shaObj = new jsSHA("SHA-256", "TEXT"); - shaObj.update(writing_lasthash); - shaObj.update(Math.random().toString()); - shaObj.update(Date.now().toString()); - shaObj.update(document.cookie); - shaObj.update("NaCl"); - shaObj.update(window.location.href); - writing_lasthash = shaObj.getHash("HEX"); - return writing_lasthash; -} - -function dequeue_events() { - while(event_queue.length > 0) { - if((webSocket == null) || (webSocket.readyState != 1) ) { - window.setTimeout(reset_websocket, 1000); - return; - } - var event = event_queue.shift(); - webSocket.send(JSON.stringify(event)); - } -} - -function reset_websocket() { - if((webSocket == null) || (webSocket.readyState != 1) ) { - webSocket = new WebSocket("wss://writing.hopto.org/wsapi/"); - webSocket.onopen = dequeue_events; - } -} - function enqueue_event(event) { if(EXPERIMENTAL_WEBSOCKET) { event_queue.push(event); @@ -86,10 +75,33 @@ function enqueue_event(event) { } } -enqueue_event({"event": "extension_loaded"}); -if(EXPERIMENTAL_WEBSOCKET) { - reset_websocket(); + + +function send_chrome_identity() { + /* + We sometimes may want to log the user's identity, as stored in + Google Chrome. Note that this is not secure; we need oauth to do + that. oauth can be distracting in that (at least in the workflow + we used), it requires the user to confirm permissions. + + Perhaps want to do oauth exactly once per device, and use a + unique token stored as a cookie or in browser.storage? + + Note this function is untested, following a refactor. + */ + chrome.identity.getProfileInfo(function(userInfo) { + enqueue_event("chrome_identity", {"email": userInfo.email, + "id": userInfo.id + }); + }); +} + +function reset_websocket() { + if((webSocket == null) || (webSocket.readyState != 1) ) { + webSocket = new WebSocket("wss://writing.hopto.org/wsapi/"); + webSocket.onopen = dequeue_events; + } } function this_a_google_docs_save(request) { @@ -106,6 +118,26 @@ function this_a_google_docs_save(request) { var RAW_DEBUG = false; // Do not save debug requests. We flip this frequently. Perhaps this should be a cookie or browser.storage? +// Figure out the system settings. Note this is asynchronous, so we +// chain dequeue_events when this is done. +chrome.storage.sync.get(['process-server'], function(result) { + //WRITINGJS_AJAX_SERVER = result['process-server']; + if(!WRITINGJS_AJAX_SERVER) { + WRITINGJS_AJAX_SERVER = "https://writing.hopto.org/webapi/"; + } + dequeue_events(); +}); + +// Listen for the keystroke messages from the page script and forward to the server. +chrome.runtime.onMessage.addListener( + function(request, sender, sendResponse) { + chrome.extension.getBackgroundPage().console.log("Got message"); + chrome.extension.getBackgroundPage().console.log(request); + enqueue_event(request); + } +); + +// Listen for web loads, and forward relevant ones (e.g. saves) to the server. chrome.webRequest.onBeforeRequest.addListener( /* This allows us to log web requests. There are two types of web requests: @@ -168,23 +200,13 @@ chrome.webRequest.onBeforeRequest.addListener( ['requestBody'] ) -chrome.runtime.onMessage.addListener( - function(request, sender, sendResponse) { - chrome.extension.getBackgroundPage().console.log("Got message"); - chrome.extension.getBackgroundPage().console.log(request); - enqueue_event(request); - } -); - - -/*chrome.tabs.executeScript({ - code: 'console.log("addd")' -});*/ - -enqueue_event({"Loaded now": true}); +// Let the server know we've loaded. +enqueue_event({"event": "extension_loaded"}); +// Send the server the user info. This might not always be available. chrome.identity.getProfileUserInfo(function callback(userInfo) { enqueue_event(userInfo); }); +// And let the console know we've loaded chrome.extension.getBackgroundPage().console.log("Loaded"); diff --git a/extension/writing_common.js b/extension/writing_common.js index 9bd7cccc1..097d38a54 100644 --- a/extension/writing_common.js +++ b/extension/writing_common.js @@ -13,3 +13,19 @@ function googledocs_id_from_url(url) { return null; } +var writing_lasthash = ""; +function unique_id() { + /* + This function is used to generate a (hopefully) unique ID for + each event. + */ + var shaObj = new jsSHA("SHA-256", "TEXT"); + shaObj.update(writing_lasthash); + shaObj.update(Math.random().toString()); + shaObj.update(Date.now().toString()); + shaObj.update(document.cookie); + shaObj.update("NaCl"); /* Salt? */ + shaObj.update(window.location.href); + writing_lasthash = shaObj.getHash("HEX"); + return writing_lasthash; +} From 50005296fbe159f9c5126200033881ea2977b6cb Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 9 Feb 2020 12:49:19 -0500 Subject: [PATCH 029/750] Syncing between machines; implementing queue --- extension/background.js | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/extension/background.js b/extension/background.js index 88bc2c6a3..3c2f476c9 100644 --- a/extension/background.js +++ b/extension/background.js @@ -14,7 +14,6 @@ var webSocket = null; //var WRITINGJS_WSS_SERVER = "https://writing.hopto.org/webapi/"; var WRITINGJS_AJAX_SERVER = null; - var EXPERIMENTAL_WEBSOCKET = false; /* @@ -38,7 +37,16 @@ Dequeue events */ function dequeue_events() { -/* while(event_queue.length > 0) { + // If we have not yet initialized, we rely on the queue to be + // flushed once we are initialized. + if(!WRITINGJS_AJAX_SERVER) { + return + } + while(event_queue.length > 0) { + writingjs_ajax(event_queue.shift()); + } + /* + if(EXPERIMENTAL_WEBSOCKET) { if((webSocket == null) || (webSocket.readyState != 1) ) { window.setTimeout(reset_websocket, 1000); return; @@ -66,18 +74,10 @@ function writingjs_ajax(data) { } function enqueue_event(event) { - if(EXPERIMENTAL_WEBSOCKET) { - event_queue.push(event); - dequeue_events(); - } - else { - writingjs_ajax(event) - } + event_queue.push(event); + dequeue_events(); } - - - function send_chrome_identity() { /* We sometimes may want to log the user's identity, as stored in From 8f06aa8f95a483f0d8919a8bd36da95f236d0e2f Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Thu, 13 Feb 2020 14:30:24 -0500 Subject: [PATCH 030/750] Readme file --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000..2bff5383e --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +# Writing analysis +This repository is for a writing analysis project. There isn't much to see here yet. + +Contact/maintainer: Piotr Mitros (pmitros@ets.org) + +Licensing: Open source / free software. License TBD. \ No newline at end of file From ca8ceac03aa4def42fd5f1ff76f6da752d270347 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Thu, 13 Feb 2020 18:51:45 -0500 Subject: [PATCH 031/750] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2bff5383e..d06243fed 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Writing analysis -This repository is for a writing analysis project. There isn't much to see here yet. +This repository is for a writing analysis project. There isn't much to see here yet. We're planning to look at writing processes in K-12 classrooms. Contact/maintainer: Piotr Mitros (pmitros@ets.org) -Licensing: Open source / free software. License TBD. \ No newline at end of file +Licensing: Open source / free software. License TBD. From e09bae17f3aedfd00ce60553a4bcfda7ca9fa839 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 5 Apr 2020 08:23:12 -0400 Subject: [PATCH 032/750] Experimental (incomplete) commit. --- configuration/tasks/writing.yaml | 1 + extension/background.js | 13 ++-- ux/deanne3.js | 113 +++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 9 deletions(-) create mode 100644 ux/deanne3.js diff --git a/configuration/tasks/writing.yaml b/configuration/tasks/writing.yaml index c178b50ee..ae760c05a 100644 --- a/configuration/tasks/writing.yaml +++ b/configuration/tasks/writing.yaml @@ -49,3 +49,4 @@ - postgresql - nginx - certbot + - ejabberd \ No newline at end of file diff --git a/extension/background.js b/extension/background.js index 3c2f476c9..ad6051843 100644 --- a/extension/background.js +++ b/extension/background.js @@ -2,19 +2,14 @@ Background script. This works across all of Google Chrome. */ - -var event_queue = []; - -/* To avoid race conditions, we keep track of events we've successfully sent */ -var sent_events = new Set(); +var WRITINGJS_AJAX_SERVER = null; var webSocket = null; +var EXPERIMENTAL_WEBSOCKET = false; -//var WRITINGJS_AJAX_SERVER = "https://writing.hopto.org/webapi/"; -//var WRITINGJS_WSS_SERVER = "https://writing.hopto.org/webapi/"; -var WRITINGJS_AJAX_SERVER = null; -var EXPERIMENTAL_WEBSOCKET = false; +var event_queue = []; + /* FSM diff --git a/ux/deanne3.js b/ux/deanne3.js new file mode 100644 index 000000000..56dc12605 --- /dev/null +++ b/ux/deanne3.js @@ -0,0 +1,113 @@ +const LENGTH = 30; + +const width = 960; +const height = 500; +const margin = 5; +const padding = 5; +const adj = 30; + +function consecutive_array(n) { + /* + This creates an array of length n [0,1,2,3,4...n] + */ + return Array(n).fill().map((e,i)=>i+1); +}; + +function randn_bm() { + /* Approximately Gaussian distribution, mean 0.5 + From https://stackoverflow.com/questions/25582882/javascript-math-random-normal-distribution-gaussian-bell-curve */ + let u = 0, v = 0; + while(u === 0) u = Math.random(); //Converting [0,1) to (0,1) + while(v === 0) v = Math.random(); + let num = Math.sqrt( -2.0 * Math.log( u ) ) * Math.cos( 2.0 * Math.PI * v ); + num = num / 10.0 + 0.5; // Translate to 0 -> 1 + if (num > 1 || num < 0) return randn_bm(); // resample between 0 and 1 + return num; +} + + +function length_array(x) { + /* + Essay length + */ + return x.map((e,i)=> (e*randn_bm(e) + e)/2); +} + +function cursor_array(x) { + /* + Essay cursor position + */ + var length_array = x.map((e,i)=> (e*Math.random()/2 + e*randn_bm()/2)); + return length_array; +} + +function zip(a1, a2) { + return a1.map(function(e, i) { + return [e, a2[i]]; + }); +} + +function make_deanne_graph(div) { + var svg = d3.select(div).append("svg") + .attr("preserveAspectRatio", "xMinYMin meet") + .attr("viewBox", "-" + + adj + " -" + + adj + " " + + (width + adj *3) + " " + + (height + adj*3)) + .style("padding", padding) + .style("margin", margin) + .style("border", "1px solid lightgray") + .classed("svg-content", true); + + var x_edit = consecutive_array(LENGTH); + var y_length = length_array(x_edit); + var y_cursor = cursor_array(y_length); + + + const yScale = d3.scaleLinear().range([height, 0]).domain([0, LENGTH]) + const xScale = d3.scaleLinear().range([0, width]).domain([0, LENGTH]) + + + var xAxis = d3.axisBottom(xScale) + .ticks(4); // specify the number of ticks + var yAxis = d3.axisLeft(yScale) + .ticks(4); // specify the number of ticks + + svg.append('g') // create a element + .attr("transform", "translate(0, "+height+")") + .attr('class', 'x axis') // specify classes + .call(xAxis); // let the axis do its thing + + svg.append('g') // create a element + .attr('class', 'y axis') // specify classes + .call(yAxis); // let the axis do its thing + + var lines = d3.line(); + + var length_data = zip(x_edit.map(xScale), y_length.map(yScale)); + + var cursor_data = zip(x_edit.map(xScale), y_cursor.map(yScale)); + + var pathData = lines(length_data); + + svg.append('g') // create a element + .attr('class', 'essay-length lines') + .append('path') + .attr('d', pathData) + .attr('fill', 'none') + .attr('stroke', 'black') + .attr('stroke-width','3'); + + pathData = lines(cursor_data); + + svg.append('g') // create a element + .attr('class', 'essay-length lines') + .append('path') + .attr('d', pathData) + .attr('fill', 'none') + .attr('stroke', 'black') + .attr('stroke-width','3'); +} + +make_deanne_graph("#deanne") From 9e2fbee0442b5b7d66fec2b93a507497616e1551 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 5 Apr 2020 12:24:19 -0400 Subject: [PATCH 033/750] Considering refactor. Saving state --- ux/deanne.html | 16 +++ ux/deanne3.js | 22 ++-- ux/media/ETS_Logo.svg | 18 +++ ux/typing.html | 15 +++ ux/typing.js | 59 ++++++++++ ux/ux.css | 31 ++++++ ux/ux.html | 249 ++++++++++++++++++++++++++++++++++++++++++ ux/writing.js | 6 + 8 files changed, 407 insertions(+), 9 deletions(-) create mode 100644 ux/deanne.html create mode 100644 ux/media/ETS_Logo.svg create mode 100644 ux/typing.html create mode 100644 ux/typing.js create mode 100644 ux/ux.css create mode 100644 ux/ux.html create mode 100644 ux/writing.js diff --git a/ux/deanne.html b/ux/deanne.html new file mode 100644 index 000000000..3ce2f4a1c --- /dev/null +++ b/ux/deanne.html @@ -0,0 +1,16 @@ + + + + + + + + +

    Deanne

    +
    +
    + + + diff --git a/ux/deanne3.js b/ux/deanne3.js index 56dc12605..84dbad5f8 100644 --- a/ux/deanne3.js +++ b/ux/deanne3.js @@ -47,8 +47,10 @@ function zip(a1, a2) { }); } -function make_deanne_graph(div) { - var svg = d3.select(div).append("svg") +export const name = 'deanne3'; + +export function deanne_graph(div) { + var svg = div.append("svg") .attr("preserveAspectRatio", "xMinYMin meet") .attr("viewBox", "-" + adj + " -" @@ -78,7 +80,7 @@ function make_deanne_graph(div) { .attr("transform", "translate(0, "+height+")") .attr('class', 'x axis') // specify classes .call(xAxis); // let the axis do its thing - + svg.append('g') // create a element .attr('class', 'y axis') // specify classes .call(yAxis); // let the axis do its thing @@ -86,11 +88,11 @@ function make_deanne_graph(div) { var lines = d3.line(); var length_data = zip(x_edit.map(xScale), y_length.map(yScale)); - + var cursor_data = zip(x_edit.map(xScale), y_cursor.map(yScale)); - + var pathData = lines(length_data); - + svg.append('g') // create a element .attr('class', 'essay-length lines') .append('path') @@ -98,9 +100,9 @@ function make_deanne_graph(div) { .attr('fill', 'none') .attr('stroke', 'black') .attr('stroke-width','3'); - + pathData = lines(cursor_data); - + svg.append('g') // create a element .attr('class', 'essay-length lines') .append('path') @@ -108,6 +110,8 @@ function make_deanne_graph(div) { .attr('fill', 'none') .attr('stroke', 'black') .attr('stroke-width','3'); + + return svg; } -make_deanne_graph("#deanne") +d3.select("#deanne").call(deanne_graph).call(console.log); diff --git a/ux/media/ETS_Logo.svg b/ux/media/ETS_Logo.svg new file mode 100644 index 000000000..e6a0a1807 --- /dev/null +++ b/ux/media/ETS_Logo.svg @@ -0,0 +1,18 @@ + + + + + + + \ No newline at end of file diff --git a/ux/typing.html b/ux/typing.html new file mode 100644 index 000000000..1253eda43 --- /dev/null +++ b/ux/typing.html @@ -0,0 +1,15 @@ + + + + + + + +

    Essay

    +

    +

    + + + diff --git a/ux/typing.js b/ux/typing.js new file mode 100644 index 000000000..4b0489a80 --- /dev/null +++ b/ux/typing.js @@ -0,0 +1,59 @@ +export const name = 'typing'; + +const SAMPLE_TEXT = "I like the goals of this petition and the bills, but as drafted, these bills just don't add up. We want to put our economy on hold. We definitely need a rent freeze. For that to work, we also need a mortgage freeze, not a mortgage forbearance. The difference is that in a mortgage forbearance, interest adds up and at the end, your principal is higher than when you started. In a mortgage freeze, the principal doesn't change -- you just literally push back all payments by a few months."; + +export function typing(div, ici=200, text=SAMPLE_TEXT) { + function randn_bm() { + /* Approximately Gaussian distribution, mean 0.5 + From https://stackoverflow.com/questions/25582882/javascript-math-random-normal-distribution-gaussian-bell-curve */ + let u = 0, v = 0; + while(u === 0) u = Math.random(); //Converting [0,1) to (0,1) + while(v === 0) v = Math.random(); + let num = Math.sqrt( -2.0 * Math.log( u ) ) * Math.cos( 2.0 * Math.PI * v ); + num = num / 10.0 + 0.5; // Translate to 0 -> 1 + if (num > 1 || num < 0) return randn_bm(); // resample between 0 and 1 + return num; + } + + function sample_ici(typing_delay=200) { + /* + Intercharacter interval -- how long between two keypresses + + We do an approximate Gaussian distribution around the + */ + return typing_delay * randn_bm() * 2; + } + + var start = 0; + var stop = 1; + const MAXIMUM_LENGTH = 300; + + function updateText() { + //document.getElementsByClassName("typing")[0].innerText=text.substr(start, stop-start); + div.text(text.substr(start, stop-start)); + stop = stop + 1; + + if(stop > text.length) { + stop = 1; + start = 0; + } + + start = Math.max(start, stop-MAXIMUM_LENGTH); + while((text[start] != ' ') && (start>0) && (startstop) { + start=stop; + } + + if(div.size() > 0) { + setTimeout(updateText, sample_ici(ici)); + }; + } + setTimeout(updateText, sample_ici(50)); +}; + +//typing(); + +d3.select(".typingdebug-typing").call(typing); + diff --git a/ux/ux.css b/ux/ux.css new file mode 100644 index 000000000..b8f336b76 --- /dev/null +++ b/ux/ux.css @@ -0,0 +1,31 @@ +/* Flip based on https://davidwalsh.name/css-flip */ +.wa-flip-container { + perspective: 1000px; +} + +.wa-flipper { + transition: 0.6s; + transform-style: preserve-3d; + + position: relative; +} + +.wa-front, .wa-back { + backface-visibility: hidden; + + position: absolute; + top: 20px; + left: 20px; +} + +/* front pane, placed above back */ +.wa-front { + z-index: 2; + /* for firefox 31 */ + transform: rotateY(0deg); +} + +/* back, initially hidden pane */ +.wa-back { + transform: rotateY(180deg); +} diff --git a/ux/ux.html b/ux/ux.html new file mode 100644 index 000000000..6e77ed1a8 --- /dev/null +++ b/ux/ux.html @@ -0,0 +1,249 @@ + + + + + + + + + + + + + + + + + + + + Writing Analysis + + + + + + + +
    +
    +
    + +
    + +
    + +
    +
    +

    Irene

    +

    Subtitle

    +

    irene@kong.ma.us
    + 408 Wrangler Lane
    + 617-889-2292

    + + + + + + + + + +
    +
    +
    +
    +

    Three

    +

    Subtitle

    +
    +
    +
    +
    +
    +
    +

    Four

    +

    Subtitle

    +
    +
    +
    + +
    +
    +
    +

    One

    +

    Subtitle

    +
    +
    +
    +
    +

    Two

    +

    Subtitle

    +
    +
    +
    +
    +

    Three

    +

    Subtitle

    +
    +
    +
    +
    +

    Four

    +

    Subtitle

    +
    +
    +
    + +
    +
    +
    +

    One

    +

    Subtitle

    +
    +
    +
    +
    +

    Two

    +

    Subtitle

    +
    +
    +
    +
    +

    Three

    +

    Subtitle

    +
    +
    +
    +
    +

    Four

    +

    Subtitle

    +
    +
    +
    + +
    +
    +
    +

    One

    +

    Subtitle

    +
    +
    +
    +
    +

    Two

    +

    Subtitle

    +
    +
    +
    +
    +

    Three

    +

    Subtitle

    +
    +
    +
    +
    +

    Four

    +

    Subtitle

    +
    +
    +
    + +
    +
    +
    +

    One

    +

    Subtitle

    +
    +
    +
    +
    +

    Two

    +

    Subtitle

    +
    +
    +
    +
    +

    Three

    +

    Subtitle

    +
    +
    +
    +
    +

    Four

    +

    Subtitle

    +
    +
    +
    + + + +
    +
    + + + + diff --git a/ux/writing.js b/ux/writing.js new file mode 100644 index 000000000..2b1900284 --- /dev/null +++ b/ux/writing.js @@ -0,0 +1,6 @@ +import { deanne_graph } from './deanne3.js' +import { typing } from './typing.js' + +d3.select(".typing").call(typing); + +console.log("Hello!"); From 13e511c5c97db1782f0dece8536e23338b041052 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Mon, 6 Apr 2020 14:30:02 -0400 Subject: [PATCH 034/750] Progress --- ux/deanne.html | 3 +- ux/deanne3.js | 2 +- ux/summary_stats.html | 15 +++++++ ux/summary_stats.js | 80 +++++++++++++++++++++++++++++++++++++ ux/typing.js | 2 +- ux/ux.css | 8 ++++ ux/ux.html | 91 ++++++++++++++++++++++++++++++------------- ux/writing.js | 52 ++++++++++++++++++++++++- 8 files changed, 219 insertions(+), 34 deletions(-) create mode 100644 ux/summary_stats.html create mode 100644 ux/summary_stats.js diff --git a/ux/deanne.html b/ux/deanne.html index 3ce2f4a1c..f1b0b039e 100644 --- a/ux/deanne.html +++ b/ux/deanne.html @@ -7,8 +7,7 @@

    Deanne

    -
    -
    +
    + + + + +

    Summary Stats

    +
    + + + diff --git a/ux/summary_stats.js b/ux/summary_stats.js new file mode 100644 index 000000000..b49596915 --- /dev/null +++ b/ux/summary_stats.js @@ -0,0 +1,80 @@ +const LENGTH = 30; + +const width = 960; +const height = 500; +const margin = 5; +const padding = 5; +const adj = 30; + +export const name = 'summary_stats'; + +export function summary_stats(div) { + var svg = div.append("svg") + .attr("preserveAspectRatio", "xMinYMin meet") + .attr("viewBox", "-" + + adj + " -" + + adj + " " + + (width + adj *3) + " " + + (height + adj*3)) + .style("padding", padding) + .style("margin", margin) + .style("border", "1px solid lightgray") + .classed("svg-content", true); + + data = { + 'Active Time': 901, + 'Date Started': 5, + 'Characters Typed': 4065, + 'Text Complexity': 8, + 'Time Since Last Edit': 3, + 'Word Count': 678 + }; + + const yScale = d3.scaleLinear().range([height, 0]).domain([0, LENGTH]) + const xScale = d3.scaleLinear().range([0, width]).domain([0, LENGTH]) + + + var xAxis = d3.axisBottom(xScale) + .ticks(4); // specify the number of ticks + var yAxis = d3.axisLeft(yScale) + .ticks(4); // specify the number of ticks + + svg.append('g') // create a element + .attr("transform", "translate(0, "+height+")") + .attr('class', 'x axis') // specify classes + .call(xAxis); // let the axis do its thing + + svg.append('g') // create a element + .attr('class', 'y axis') // specify classes + .call(yAxis); // let the axis do its thing + + var lines = d3.line(); + + var length_data = zip(x_edit.map(xScale), y_length.map(yScale)); + + var cursor_data = zip(x_edit.map(xScale), y_cursor.map(yScale)); + + var pathData = lines(length_data); + + svg.append('g') // create a element + .attr('class', 'essay-length lines') + .append('path') + .attr('d', pathData) + .attr('fill', 'none') + .attr('stroke', 'black') + .attr('stroke-width','3'); + + pathData = lines(cursor_data); + + svg.append('g') // create a element + .attr('class', 'essay-length lines') + .append('path') + .attr('d', pathData) + .attr('fill', 'none') + .attr('stroke', 'black') + .attr('stroke-width','3'); + + return svg; +} + +d3.select("#debug_testing_summary").call(summary_stats).call(console.log); diff --git a/ux/typing.js b/ux/typing.js index 4b0489a80..e9f30436f 100644 --- a/ux/typing.js +++ b/ux/typing.js @@ -26,7 +26,7 @@ export function typing(div, ici=200, text=SAMPLE_TEXT) { var start = 0; var stop = 1; - const MAXIMUM_LENGTH = 300; + const MAXIMUM_LENGTH = 250; function updateText() { //document.getElementsByClassName("typing")[0].innerText=text.substr(start, stop-start); diff --git a/ux/ux.css b/ux/ux.css index b8f336b76..29030635f 100644 --- a/ux/ux.css +++ b/ux/ux.css @@ -1,3 +1,11 @@ +.wa-row-tile { + min-height: 350px; +} + +.wa-col-tile { + min-height: 350px; +} + /* Flip based on https://davidwalsh.name/css-flip */ .wa-flip-container { perspective: 1000px; diff --git a/ux/ux.html b/ux/ux.html index 6e77ed1a8..c2eb5918c 100644 --- a/ux/ux.html +++ b/ux/ux.html @@ -19,9 +19,47 @@ Writing Analysis - + + + + + + + + + + + - +
    - + +
    +
    diff --git a/ux/writing.js b/ux/writing.js index 2b1900284..3e9937826 100644 --- a/ux/writing.js +++ b/ux/writing.js @@ -1,6 +1,54 @@ import { deanne_graph } from './deanne3.js' import { typing } from './typing.js' -d3.select(".typing").call(typing); +var student_data = [[1,2,3,4],[5,6,7,8],[9,10,11,12],[13,14,15,16],[17,18,19]]; -console.log("Hello!"); +const tile_template = document.getElementById('template-tile').innerHTML + +function populate_tiles(tilesheet) { + var rows=tilesheet.selectAll("div.wa-row-tile") + .data(student_data) + .enter() + .append("div") + .attr("class", "tile is-ancestor wa-row-tile"); + + var cols=rows.selectAll("div.wa-col-tile") + .data(function(d) { return d; }) + .enter() + .append("div") + .attr("class", "tile is-parent wa-col-tile wa-flip-container is-3") + .html(function(d) { + return Mustache.render(tile_template, d); + /*{ + name: d.name, + body: document.getElementById('template-deanne-tile').innerHTML + });*/ + }) + .each(function(d) { + d3.select(this).select(".typing-text").call(typing, d.ici, d.essay); + }) + .each(function(d) { + d3.select(this).select(".deanne").call(deanne_graph); + }); +} + +function select_tab(tab) { + return function() { + d3.selectAll(".tilenav").classed("is-active", false); + d3.selectAll(".tilenav-"+tab).classed("is-active", true); + d3.selectAll(".wa-tilebody").classed("is-hidden", true); + d3.selectAll("."+tab).classed("is-hidden", false); + } +}; + +var tabs = ["typing", "deanne", "summary", "outline", "revision", "contact"]; +for(var i=0; i Date: Mon, 6 Apr 2020 16:57:04 -0400 Subject: [PATCH 035/750] Placeholders --- ux/ux.html | 32 +++++++++++++------------------- ux/writing.js | 2 +- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/ux/ux.html b/ux/ux.html index c2eb5918c..4d481c67f 100644 --- a/ux/ux.html +++ b/ux/ux.html @@ -25,18 +25,27 @@
diff --git a/ux/writing.js b/ux/writing.js index 3e9937826..4651ed79b 100644 --- a/ux/writing.js +++ b/ux/writing.js @@ -41,7 +41,7 @@ function select_tab(tab) { } }; -var tabs = ["typing", "deanne", "summary", "outline", "revision", "contact"]; +var tabs = ["typing", "deanne", "summary", "outline", "timeline", "contact"]; for(var i=0; i Date: Mon, 6 Apr 2020 22:38:38 -0400 Subject: [PATCH 036/750] Summary stats ugly but works, outline starting --- ux/outline.html | 15 +++++ ux/outline.js | 67 ++++++++++++++++++++ ux/summary_stats.js | 150 ++++++++++++++++++++++++++++---------------- ux/ux.html | 1 - ux/writing.js | 8 +++ 5 files changed, 186 insertions(+), 55 deletions(-) create mode 100644 ux/outline.html create mode 100644 ux/outline.js diff --git a/ux/outline.html b/ux/outline.html new file mode 100644 index 000000000..c2ff6663b --- /dev/null +++ b/ux/outline.html @@ -0,0 +1,15 @@ + + + + + + + + +

Outline

+
+ + + diff --git a/ux/outline.js b/ux/outline.js new file mode 100644 index 000000000..f54faa635 --- /dev/null +++ b/ux/outline.js @@ -0,0 +1,67 @@ +const LENGTH = 30; + +const width = 960; +const height = 650; +const margin = 5; +const padding = 5; +const adj = 30; + +export const name = 'outline'; + +var test_data = { "outline": [ + ["Problem 1", 300], + ["Problem 2", 30], + ["Problem 3", 900], + ["Problem 4", 1200], + ["Problem 5", 400] +]}; + +var maximum = 1500; + +export function outline(div, data=test_data) { + div.html(""); + var svg = div.append("svg") + .attr("preserveAspectRatio", "xMinYMin meet") + .attr("viewBox", "-" + + adj + " -" + + adj + " " + + (width + adj *3) + " " + + (height + adj*3)) + .style("padding", padding) + .style("margin", margin) + .style("border", "1px solid lightgray") + .classed("svg-content", true); + + console.log(data.outline); + + var outline_data = data.outline; + const yScale = d3.scaleBand().range([height, 0]).domain(outline_data.map(d=>d[0])); + const xScale = d3.scaleLinear().range([0, width]).domain([0, 1]) + + var normed_x = (x) => x / maximum; + + svg.selectAll(".barRect") + .data(outline_data) + .enter() + .append("rect") + .attr("x", (d) => xScale(1-normed_x(d[1]))) + .attr("y", function(d) { return yScale(d[0]);}) + .attr("width", function(d) { return xScale(normed_x(d[1]));}) + .attr("height", yScale.bandwidth()) + .attr("fill", "#ccccff"); + + svg.selectAll(".barText") + .data(outline_data) + .enter() + .append("text") + .attr("x", 0) + .attr("y", (d) => yScale(d[0]) + yScale.bandwidth()/2) + .attr("font-size", "3.5em") + .attr("font-family", 'BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif') + .text((d) => d[0]) + .call((d) => console.log(d)); + + return svg; +} + +d3.select("#debug_testing_outline").call(outline).call(console.log); diff --git a/ux/summary_stats.js b/ux/summary_stats.js index b49596915..56cc5c2c5 100644 --- a/ux/summary_stats.js +++ b/ux/summary_stats.js @@ -1,14 +1,40 @@ const LENGTH = 30; const width = 960; -const height = 500; +const height = 650; const margin = 5; const padding = 5; const adj = 30; export const name = 'summary_stats'; -export function summary_stats(div) { +var bar_names = { + "speed": "Typing speed", + "essay_length": "Length", + "writing_time": "Writing time", + "text_complexity": "Text complexity", + "time_idle": "Time idle" +}; + +var maxima = { + "ici": 1000, + "speed": 1300, + "essay_length": 10000, + "writing_time": 60, + "text_complexity": 12, + "time_idle": 30 +} + +var test_data = { + "ici": 729.664923175084, + "essay_length": 2221, + "writing_time": 42.05237247614963, + "text_complexity": 4.002656228025943, + "time_idle": 0.24548328432300075 +}; + + +export function summary_stats(div, data=test_data) { var svg = div.append("svg") .attr("preserveAspectRatio", "xMinYMin meet") .attr("viewBox", "-" @@ -21,59 +47,75 @@ export function summary_stats(div) { .style("border", "1px solid lightgray") .classed("svg-content", true); - data = { - 'Active Time': 901, - 'Date Started': 5, - 'Characters Typed': 4065, - 'Text Complexity': 8, - 'Time Since Last Edit': 3, - 'Word Count': 678 - }; - - const yScale = d3.scaleLinear().range([height, 0]).domain([0, LENGTH]) - const xScale = d3.scaleLinear().range([0, width]).domain([0, LENGTH]) - - - var xAxis = d3.axisBottom(xScale) - .ticks(4); // specify the number of ticks - var yAxis = d3.axisLeft(yScale) - .ticks(4); // specify the number of ticks - - svg.append('g') // create a element - .attr("transform", "translate(0, "+height+")") - .attr('class', 'x axis') // specify classes - .call(xAxis); // let the axis do its thing - - svg.append('g') // create a element - .attr('class', 'y axis') // specify classes - .call(yAxis); // let the axis do its thing - - var lines = d3.line(); - - var length_data = zip(x_edit.map(xScale), y_length.map(yScale)); - - var cursor_data = zip(x_edit.map(xScale), y_cursor.map(yScale)); - - var pathData = lines(length_data); - - svg.append('g') // create a element - .attr('class', 'essay-length lines') - .append('path') - .attr('d', pathData) - .attr('fill', 'none') - .attr('stroke', 'black') - .attr('stroke-width','3'); - - pathData = lines(cursor_data); - - svg.append('g') // create a element - .attr('class', 'essay-length lines') - .append('path') - .attr('d', pathData) - .attr('fill', 'none') - .attr('stroke', 'black') - .attr('stroke-width','3'); + data['speed'] = 60000 / data['ici'] + var data_ordered = [ + ['essay_length', data['essay_length']], + ['time_idle', data['time_idle']], + ['writing_time', data['writing_time']], + ['text_complexity', data['text_complexity']], + ['speed', data['speed']] + ]; + + const yScale = d3.scaleBand().range([height, 0]).domain(data_ordered.map(d=>d[0])); //labels); + const xScale = d3.scaleLinear().range([0, width]).domain([0, 1]) + + var y = (d) => data[d]; + var normed_x = (x) => data[x] / maxima[x]; + + function rendertime(t) { + function str(i) { + if(i<10) { + return "0"+String(i); + } + return String(i) + } + var seconds = Math.floor((t - Math.floor(t)) * 60); + var minutes = Math.floor(t) % 60; + var hours = Math.floor(t/60) % 60; + var rendered = str(seconds); + if (minutes>0 || hours>0) { + rendered = str(minutes)+":"+rendered; + } else { + rendered = rendered + " sec"; + } + if (hours>0) { + rendered = str(rendered)+":"+rendered; + } + return rendered + } + + function label(d) { + var prettyprint = { + 'essay_length': (d) => String(d) +" characters", + 'time_idle': rendertime, + 'writing_time': rendertime, + 'text_complexity': Math.floor, + 'speed': (d) => Math.floor(d) + " CPM" + } + return bar_names[d[0]] + ": " + prettyprint[d[0]](String(d[1])); + } + + svg.selectAll(".barRect") + .data(data_ordered) + .enter() + .append("rect") + .attr("x", xScale(0)) + .attr("y", function(d) { return yScale(d[0]);}) + .attr("width", function(d) { return xScale(normed_x(d[0]));}) + .attr("height", yScale.bandwidth()) + .attr("fill", "#ccccff") + + svg.selectAll(".barText") + .data(data_ordered) + .enter() + .append("text") + .attr("x", 0) + .attr("y", (d) => yScale(d[0]) + yScale.bandwidth()/2) + .attr("font-size", "3.5em") + .attr("font-family", 'BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif') + .text((d) => label(d)) + ; return svg; } diff --git a/ux/ux.html b/ux/ux.html index 4d481c67f..a3524e4da 100644 --- a/ux/ux.html +++ b/ux/ux.html @@ -35,7 +35,6 @@ -

@@ -176,6 +175,7 @@

+ diff --git a/webapp/static/dashboard.js b/webapp/static/dashboard.js index da1882331..0e5685050 100644 --- a/webapp/static/dashboard.js +++ b/webapp/static/dashboard.js @@ -65,16 +65,20 @@ var ws = new WebSocket(`wss://${window.location.hostname}/wsapi/student-data/`) ws.onmessage = function (event) { console.log("Got data"); let data = JSON.parse(event.data); - student_data = data; //.student_data; - if(data.loggedin === false) { + // dispatch + if(data.logged_in === false) { d3.selectAll(".loading").classed("is-hidden", true); d3.selectAll(".auth-form").classed("is-hidden", false); d3.selectAll(".main").classed("is-hidden", true); - } else { + } else if (data.new_student_data) { + student_data = data.new_student_data; d3.selectAll(".loading").classed("is-hidden", true); d3.selectAll(".auth-form").classed("is-hidden", true); d3.selectAll(".main").classed("is-hidden", false); d3.select(".wa-tile-sheet").html(""); d3.select(".wa-tile-sheet").call(populate_tiles); + } else { + console.log(data); + console.log("Unrecognized JSON"); } }; diff --git a/webapp/student_data.py b/webapp/student_data.py index 7fe7e27cc..5cbbd580a 100644 --- a/webapp/student_data.py +++ b/webapp/student_data.py @@ -4,24 +4,42 @@ import synthetic_student_data +def authenticated(request): + ''' + Dummy function to tell if a request is logged in + ''' + return True -def static_student_data_handler(request): + +async def static_student_data_handler(request): ''' Populate static / mock-up dashboard with static fake data ''' - return aiohttp.web.json_response(json.load(open("static/student_data.js"))) + return aiohttp.web.json_response({ + "new_student_data": json.load(open("static/student_data.js")) + }) -def generated_student_data_handler(request): +async def generated_student_data_handler(request): ''' Populate static / mock-up dashboard with static fake data dynamically ''' - return aiohttp.web.json_response(synthetic_student_data.synthetic_data()) + return aiohttp.web.json_response({ + "new_student_data": synthetic_student_data.synthetic_data() + }) + async def ws_student_data_handler(request): print("Serving") ws = aiohttp.web.WebSocketResponse() await ws.prepare(request) - await ws.send_json(synthetic_student_data.synthetic_data()) + if authenticated(request): + await ws.send_json({ + "new_student_data": synthetic_student_data.synthetic_data() + }) + else: + await ws.send_json({ + "logged_in": False + }) student_data_handler = generated_student_data_handler From a03e2d2ec82a39295ab5a0abf05c9ee7d2af99e1 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Tue, 9 Jun 2020 02:30:19 +0000 Subject: [PATCH 096/750] We can get data for many students. To confirm performance issues, etc. Also, cleanups for dynamic assessment. --- requirements.txt | 2 +- webapp/event_pipeline.py | 4 +- webapp/static/dashboard.js | 19 +++--- webapp/static/deane.js | 9 ++- webapp/static/typing.js | 9 +++ webapp/stream_analytics/helpers.py | 16 ++++- webapp/stream_writing.py | 3 +- webapp/student_data.py | 96 +++++++++++++++++++++++++++++- 8 files changed, 137 insertions(+), 21 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3e63c6acb..a147c19b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,4 +21,4 @@ docopt pathvalidate names git+https://github.com/testlabauto/loremipsum.git@b7bd71a6651207ef88993045cd755f20747f2a1e#egg=loremipsmum -git+https://github.com/pmitros/tsvx.git@57efd1ab61114650326c5acdece2f850e86b2831#egg=tsvx +git+https://github.com/pmitros/tsvx.git@09bf7f33107f66413d929075a8b54c36ca581dae#egg=tsvx diff --git a/webapp/event_pipeline.py b/webapp/event_pipeline.py index ea032d3bb..b99d19119 100644 --- a/webapp/event_pipeline.py +++ b/webapp/event_pipeline.py @@ -239,8 +239,8 @@ async def incoming_websocket_handler(request): headers["test_framework_fake_identity"] = json_msg["user_id"] event_metadata['headers'].update(headers) - event_metadata['auth'] = await auth(headers) - print(event_metadata) + event_metadata['auth'] = await auth(headers) + print(event_metadata) event_handler = await handle_incoming_client_event(metadata=event_metadata) diff --git a/webapp/static/dashboard.js b/webapp/static/dashboard.js index 0e5685050..7d928d135 100644 --- a/webapp/static/dashboard.js +++ b/webapp/static/dashboard.js @@ -1,5 +1,5 @@ import { deane_graph } from './deane.js' -import { typing } from './typing.js' +import { student_text } from './text.js' import { summary_stats } from './summary_stats.js' import { outline } from './outline.js' @@ -27,10 +27,14 @@ function populate_tiles(tilesheet) { });*/ }) .each(function(d) { - d3.select(this).select(".typing-text").call(typing, d.ici, d.essay); + d3.select(this).select(".typing-text").call( + student_text, + d["stream_analytics.writing_analysis.reconstruct"].text); }) .each(function(d) { - d3.select(this).select(".deane").call(deane_graph); + d3.select(this).select(".deane").call( + deane_graph, + d["stream_analytics.writing_analysis.reconstruct"].edit_metadata); }) .each(function(d) { d3.select(this).select(".summary").call(summary_stats, d); @@ -54,23 +58,18 @@ for(var i=0; i Date: Sun, 19 Jul 2020 16:42:59 +0000 Subject: [PATCH 097/750] Basic Google auth --- webapp/aio_webapp_w.py | 42 ++++++- webapp/auth_handlers.py | 239 +++++++++++++++++++++++++++++++++++++++ webapp/static/index.html | 48 ++++++++ webapp/static/text.js | 36 ++++++ 4 files changed, 359 insertions(+), 6 deletions(-) create mode 100644 webapp/auth_handlers.py create mode 100644 webapp/static/index.html create mode 100644 webapp/static/text.js diff --git a/webapp/aio_webapp_w.py b/webapp/aio_webapp_w.py index 3ce9edbd9..a589f9ec4 100644 --- a/webapp/aio_webapp_w.py +++ b/webapp/aio_webapp_w.py @@ -11,12 +11,19 @@ import aiohttp import aiohttp_cors +import aiohttp_session +import aiohttp_session.cookie_storage + +import hashlib import pathvalidate import init # Odd import which makes sure we're set up import event_pipeline import student_data +import auth_handlers + +import settings routes = aiohttp.web.RouteTableDef() app = aiohttp.web.Application() @@ -32,11 +39,20 @@ async def request_logger_middleware(request, handler): app.on_response_prepare.append(request_logger_middleware) -def static_file_handler(basepath): +def static_file_handler(filename): ''' - Serve static files. + Serve a single static file + ''' + def handler(request): + return aiohttp.web.FileResponse(filename) + return handler + - This can be done directly by nginx on deployment. +def static_directory_handler(basepath): + ''' + Serve static files from a directory. + + This could be done directly by nginx on deployment. This is very minimal for now: No subdirectories, no gizmos, nothing fancy. I avoid fancy when we have user input and @@ -66,10 +82,10 @@ def handler(request): # Serve static files app.add_routes([ - aiohttp.web.get('/static/{filename}', static_file_handler("static")), - aiohttp.web.get('/static/media/{filename}', static_file_handler("media")), + aiohttp.web.get('/static/{filename}', static_directory_handler("static")), + aiohttp.web.get('/static/media/{filename}', static_directory_handler("media")), aiohttp.web.get('/static/media/avatar/{filename}', - static_file_handler("media/hubspot_persona_images/")), + static_directory_handler("media/hubspot_persona_images/")), ]) # Handle web sockets event requests, incoming and outgoing @@ -84,6 +100,11 @@ def handler(request): aiohttp.web.post('/webapi/event/', event_pipeline.ajax_event_request), ]) +app.add_routes([ + aiohttp.web.get('/', static_file_handler("static/index.html")), + aiohttp.web.get('/auth/login/{provider:google}', handler=auth_handlers.social) +]) + cors = aiohttp_cors.setup(app, defaults={ "*": aiohttp_cors.ResourceOptions( allow_credentials=True, @@ -92,4 +113,13 @@ def handler(request): ) }) +def fernet_key(s): + t = hashlib.md5() + t.update(s.encode('utf-8')) + return t.hexdigest().encode('utf-8') + +aiohttp_session.setup(app, aiohttp_session.cookie_storage.EncryptedCookieStorage( + fernet_key(settings.settings['aio']['session_secret']), + max_age=settings.settings['aio']['session_max_age'])) + aiohttp.web.run_app(app, port=8888) diff --git a/webapp/auth_handlers.py b/webapp/auth_handlers.py new file mode 100644 index 000000000..576ce190f --- /dev/null +++ b/webapp/auth_handlers.py @@ -0,0 +1,239 @@ +"""Authentication for Google. + +This was based on +[aiohttp-login](https://github.com/imbolc/aiohttp-login/), which at +the time worked with outdated Google APIs and require Jinja2. Oren +modernized this. Piotr integrated this into the system. + +Portions of this file, from aiohttp-login, are licensed as: + +Copyright (c) 2011 Imbolc. + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Eventually, this should be broken out into its own module. +""" +import aiohttp +import aiohttp.web + +import logging +import os.path +import settings +#from aiohttp_jinja2 import template +#from aiohttp.abc import AbstractView +import aiohttp_session +#from functools import wraps +from yarl import URL + + + +async def social(request): + """Handles Google sign in. + + Provider is in `request.match_info['provider']` (currently, only Google) + """ + if request.match_info['provider'] != 'google': + raise aiohttp.web.HTTPMethodNotAllowed("We only handle Google logins") + + user = await _google(request) + + if 'user_id' in user: + # User ID returned in 'data', authorize user. + await _authorize_user(request, user) + url = user['back_to'] or "/" + return aiohttp.web.HTTPFound(url) + + return aiohttp.web.Response(text="Hello, world") + # No user ID returned from provider, Login failed. + #log.info(cfg['MSG_AUTH_FAILED']) + #return _redirect('auth_login') + + +# def user_to_request(handler): +# """ +# A handler function decorator that adds user to request if user logged in. +# :param handler: function to decorate. +# :return: decorated function +# """ +# @wraps(handler) +# async def decorator(*args): +# request = _get_request(args) +# request[cfg['REQUEST_USER_KEY']] = await _get_cur_user(request) +# return await handler(*args) +# return decorator + + +# def login_required(handler): +# """ +# A handler function decorator that enforces that the user is logged in. If not, redirects to the login page. +# :param handler: function to decorate. +# :return: decorated function +# """ +# @user_to_request +# @wraps(handler) +# async def decorator(*args): +# request = _get_request(args) +# if not request[cfg.REQUEST_USER_KEY]: +# return _redirect(_get_login_url(request)) +# return await handler(*args) +# return decorator + + +# @user_to_request +# @template('index.html') +# async def index(request): +# """Web app home page.""" +# return { +# 'auth': {'cfg': cfg}, +# 'cur_user': request['user'], +# 'url_for': _url_for, +# } + + +# @login_required +# @template('users.html') +# async def users(request): +# """Handles an example private page that requires logging in.""" +# return {} + + + + +# async def logout(request): +# """Handles sign out. This is generic - does not depend on which social ID is logged in +# (Google/Facebook/...).""" +# session = await aiohttp_session.get_session(request) +# session.pop(cfg["SESSION_USER_KEY"], None) +# return _redirect(cfg['LOGOUT_REDIRECT']) + + +async def _authorize_user(request, user): + """ + Logs a user in. + :param request: web request. + :param user_id: provider's user ID (e.g., Google ID). + """ + session = await aiohttp_session.get_session(request) + session["user_id"] = user + + +async def _google(request): + ''' + Handle Google login + ''' + if 'error' in request.query: + return {} + + common_params = { + 'client_id': settings.settings['google-oauth']['web']['client_id'], + 'redirect_uri': "https://writing.hopto.org/auth/login/google", + } + + # Step 1: redirect to get code + if 'code' not in request.query: + print("Here") + url = 'https://accounts.google.com/o/oauth2/auth' + params = common_params.copy() + params.update({ + 'response_type': 'code', + 'scope': ('https://www.googleapis.com/auth/userinfo.profile' + ' https://www.googleapis.com/auth/userinfo.email'), + }) + if 'back_to' in request.query: + params['state'] = request.query[back_to] + url = URL(url).with_query(params) + print(url) + raise aiohttp.web.HTTPFound(url) + + print("There") + # Step 2: get access token + url = 'https://accounts.google.com/o/oauth2/token' + params = common_params.copy() + params.update({ + 'client_secret': settings.settings['google-oauth']['web']['client_secret'], + 'code': request.query['code'], + 'grant_type': 'authorization_code', + }) + async with aiohttp.ClientSession(loop=request.app.loop) as client: + async with client.post(url, data=params) as resp: + data = await resp.json() + assert 'access_token' in data, data + + # get user profile + headers = {'Authorization': 'Bearer ' + data['access_token']} + # Old G+ URL that's no longer supported. + url = 'https://www.googleapis.com/oauth2/v1/userinfo' + async with client.get(url, headers=headers) as resp: + profile = await resp.json() + + return { + 'user_id': profile['id'], + 'email': profile['email'], + 'name': profile['given_name'], + 'family_name': profile['family_name'], + 'back_to': request.query.get('state'), + 'picture': profile['picture'], + } + + +# def _get_login_url(request): +# return _url_for('auth_login').with_query({ +# cfg['BACK_URL_QS_KEY']: request.path_qs}) + + +# async def _get_cur_user(request): +# user = await _get_cur_user_id(request) +# if user: +# # Load user object from database by the session user's user_id. This is disabled here, uncomment when we have +# # an underlying database. +# #user = await cfg.STORAGE.get_user({'id': user_id}) +# if not user: +# session = await aiohttp_session.get_session(request) +# del session['user'] +# return user + + +# async def _get_cur_user_id(request): +# session = await aiohttp_session.get_session(request) +# user = session.get(cfg['SESSION_USER_KEY']) +# while user: +# if not isinstance(user, dict): +# log.error('Wrong type of user_id in session') +# break + +# # Get a user ID from the user object. For now, we don't have a user database, so the session user is the same +# # as the "database-loaded" object user. Uncomment when we have a database. +# # user_id = cfg.STORAGE.user_id_from_string(user.user_id) +# # if not user_id: +# # break +# return user + +# if cfg['SESSION_USER_KEY'] in session: +# del session['user'] + + +# def _url_for(url_name, *args, **kwargs): +# if str(url_name).startswith(('/', 'http://', 'https://')): +# return url_name +# return cfg["APP"].router[url_name].url_for(*args, **kwargs) + + +# def _redirect(urlname, *args, **kwargs): +# return aiohttp.web.HTTPFound(_url_for(urlname, *args, **kwargs)) + + +# def _get_request(args): +# # Supports class based views see web.View +# if isinstance(args[0], AbstractView): +# return args[0].request +# return args[-1] diff --git a/webapp/static/index.html b/webapp/static/index.html new file mode 100644 index 000000000..d671f59c6 --- /dev/null +++ b/webapp/static/index.html @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + Writing Analysis + + + + +
+
+

Hello!

+ + + +
+
+
+
+

+ Writing Dashboard + by Piotr Mitros. Copyright + (c) 2020. Educational Testing + Service. The source + code will be released as free / open source software, + most likely + under the + AGPLv3 license. Our privacy policy. +

+
+
+ + diff --git a/webapp/static/text.js b/webapp/static/text.js new file mode 100644 index 000000000..557984405 --- /dev/null +++ b/webapp/static/text.js @@ -0,0 +1,36 @@ +export const name = 'student_text'; + +export function student_text(div, text) { + var start = 0; + var stop = 1; + const MAXIMUM_LENGTH = 250; + + div.text(text); +/* + .substr(start, stop-start)); + stop = stop + 1; + + if(stop > text.length) { + stop = 1; + start = 0; + } + + start = Math.max(start, stop-MAXIMUM_LENGTH); + while((text[start] != ' ') && (start>0) && (startstop) { + start=stop; + } + + if(div.size() > 0) { + setTimeout(updateText, sample_ici(ici)); + }; + } + setTimeout(updateText, sample_ici(50));*/ +} + +//typing(); + +d3.select(".textdebug-text").call(student_text); + From cad120a051f9520d1e36557cc4e5e0e016da4cc5 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 19 Jul 2020 12:44:57 -0400 Subject: [PATCH 098/750] Flags, minor changes --- configuration/local.yaml | 2 +- .../tasks/{writing.yaml => writing-apt.yaml} | 0 webapp/media/Flag_of_Poland.svg | 1 + webapp/media/Flag_of_the_United_States.svg | 26 +++++++++++++++++++ webapp/media/LICENSE.txt | 2 ++ 5 files changed, 30 insertions(+), 1 deletion(-) rename configuration/tasks/{writing.yaml => writing-apt.yaml} (100%) create mode 100644 webapp/media/Flag_of_Poland.svg create mode 100644 webapp/media/Flag_of_the_United_States.svg diff --git a/configuration/local.yaml b/configuration/local.yaml index d12102f87..3d502ca93 100644 --- a/configuration/local.yaml +++ b/configuration/local.yaml @@ -2,5 +2,5 @@ hosts: localhost connection: local tasks: - - include: tasks/writing.yaml + - include: tasks/writing-apt.yaml diff --git a/configuration/tasks/writing.yaml b/configuration/tasks/writing-apt.yaml similarity index 100% rename from configuration/tasks/writing.yaml rename to configuration/tasks/writing-apt.yaml diff --git a/webapp/media/Flag_of_Poland.svg b/webapp/media/Flag_of_Poland.svg new file mode 100644 index 000000000..b08d02519 --- /dev/null +++ b/webapp/media/Flag_of_Poland.svg @@ -0,0 +1 @@ + diff --git a/webapp/media/Flag_of_the_United_States.svg b/webapp/media/Flag_of_the_United_States.svg new file mode 100644 index 000000000..a11cf5f94 --- /dev/null +++ b/webapp/media/Flag_of_the_United_States.svg @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/webapp/media/LICENSE.txt b/webapp/media/LICENSE.txt index 6b40ec588..c3d2d6697 100644 --- a/webapp/media/LICENSE.txt +++ b/webapp/media/LICENSE.txt @@ -7,3 +7,5 @@ https://www.ets.org/legal/trademarks/owned It is not distributed under the same license as the rest of this system. + +Flags of Poland and the US are SVGs from Wikipedia, and in the public domain From 1f4430b6088b6a3e41755c6f07422b32b1ba9cb7 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Fri, 7 Aug 2020 01:58:56 +0000 Subject: [PATCH 099/750] auth works --- webapp/aio_webapp_w.py | 11 ++- webapp/auth_handlers.py | 175 ++++++++++++++-------------------------- 2 files changed, 71 insertions(+), 115 deletions(-) diff --git a/webapp/aio_webapp_w.py b/webapp/aio_webapp_w.py index a589f9ec4..c392f14c4 100644 --- a/webapp/aio_webapp_w.py +++ b/webapp/aio_webapp_w.py @@ -43,7 +43,10 @@ def static_file_handler(filename): ''' Serve a single static file ''' - def handler(request): + @auth_handlers.user_to_request + async def handler(request): + foo = await aiohttp_session.get_session(request) + print("Foo:", foo.get("user", {})) return aiohttp.web.FileResponse(filename) return handler @@ -75,6 +78,7 @@ def handler(request): # Student data API +# This serves up data (currently usually dummy data) for the dashboard app.add_routes([ aiohttp.web.get('/webapi/student-data/', student_data.student_data_handler), aiohttp.web.get('/wsapi/student-data/', student_data.ws_student_data_handler) @@ -100,9 +104,12 @@ def handler(request): aiohttp.web.post('/webapi/event/', event_pipeline.ajax_event_request), ]) +# Generic web-appy things app.add_routes([ aiohttp.web.get('/', static_file_handler("static/index.html")), - aiohttp.web.get('/auth/login/{provider:google}', handler=auth_handlers.social) + aiohttp.web.get('/auth/login/{provider:google}', handler=auth_handlers.social), + aiohttp.web.get('/auth/logout', handler=auth_handlers.logout), + aiohttp.web.get('/auth/userinfo', handler=auth_handlers.user_info) ]) cors = aiohttp_cors.setup(app, defaults={ diff --git a/webapp/auth_handlers.py b/webapp/auth_handlers.py index 576ce190f..fed270f50 100644 --- a/webapp/auth_handlers.py +++ b/webapp/auth_handlers.py @@ -32,11 +32,10 @@ #from aiohttp_jinja2 import template #from aiohttp.abc import AbstractView import aiohttp_session -#from functools import wraps +from functools import wraps from yarl import URL - async def social(request): """Handles Google sign in. @@ -46,6 +45,7 @@ async def social(request): raise aiohttp.web.HTTPMethodNotAllowed("We only handle Google logins") user = await _google(request) + print(user) if 'user_id' in user: # User ID returned in 'data', authorize user. @@ -53,68 +53,8 @@ async def social(request): url = user['back_to'] or "/" return aiohttp.web.HTTPFound(url) - return aiohttp.web.Response(text="Hello, world") - # No user ID returned from provider, Login failed. - #log.info(cfg['MSG_AUTH_FAILED']) - #return _redirect('auth_login') - - -# def user_to_request(handler): -# """ -# A handler function decorator that adds user to request if user logged in. -# :param handler: function to decorate. -# :return: decorated function -# """ -# @wraps(handler) -# async def decorator(*args): -# request = _get_request(args) -# request[cfg['REQUEST_USER_KEY']] = await _get_cur_user(request) -# return await handler(*args) -# return decorator - - -# def login_required(handler): -# """ -# A handler function decorator that enforces that the user is logged in. If not, redirects to the login page. -# :param handler: function to decorate. -# :return: decorated function -# """ -# @user_to_request -# @wraps(handler) -# async def decorator(*args): -# request = _get_request(args) -# if not request[cfg.REQUEST_USER_KEY]: -# return _redirect(_get_login_url(request)) -# return await handler(*args) -# return decorator - - -# @user_to_request -# @template('index.html') -# async def index(request): -# """Web app home page.""" -# return { -# 'auth': {'cfg': cfg}, -# 'cur_user': request['user'], -# 'url_for': _url_for, -# } - - -# @login_required -# @template('users.html') -# async def users(request): -# """Handles an example private page that requires logging in.""" -# return {} - - - - -# async def logout(request): -# """Handles sign out. This is generic - does not depend on which social ID is logged in -# (Google/Facebook/...).""" -# session = await aiohttp_session.get_session(request) -# session.pop(cfg["SESSION_USER_KEY"], None) -# return _redirect(cfg['LOGOUT_REDIRECT']) + ## Login failed. TODO: Make a proper login failed page. + return aiohttp.web.HTTPFound("/") async def _authorize_user(request, user): @@ -124,7 +64,36 @@ async def _authorize_user(request, user): :param user_id: provider's user ID (e.g., Google ID). """ session = await aiohttp_session.get_session(request) - session["user_id"] = user + session["user"] = user + + +async def logout(request): + """Handles sign out. This is generic - does not depend on which social ID is logged in + (Google/Facebook/...).""" + session = await aiohttp_session.get_session(request) + session.pop("user", None) + return aiohttp.web.HTTPFound("/") ## TODO: Make a proper logout page + + + +def user_to_request(handler): + """ + A handler function decorator that adds user to request if user logged in. + :param handler: function to decorate. + :return: decorated function + """ + @wraps(handler) + async def decorator(*args): + request = args[0] + session = await aiohttp_session.get_session(request) + request['user'] = session.get('user', None) + return await handler(*args) + return decorator + + +@user_to_request +async def user_info(request): + return aiohttp.web.json_response(request['user']) async def _google(request): @@ -141,7 +110,6 @@ async def _google(request): # Step 1: redirect to get code if 'code' not in request.query: - print("Here") url = 'https://accounts.google.com/o/oauth2/auth' params = common_params.copy() params.update({ @@ -152,10 +120,8 @@ async def _google(request): if 'back_to' in request.query: params['state'] = request.query[back_to] url = URL(url).with_query(params) - print(url) raise aiohttp.web.HTTPFound(url) - print("There") # Step 2: get access token url = 'https://accounts.google.com/o/oauth2/token' params = common_params.copy() @@ -186,54 +152,37 @@ async def _google(request): } -# def _get_login_url(request): -# return _url_for('auth_login').with_query({ -# cfg['BACK_URL_QS_KEY']: request.path_qs}) - - -# async def _get_cur_user(request): -# user = await _get_cur_user_id(request) -# if user: -# # Load user object from database by the session user's user_id. This is disabled here, uncomment when we have -# # an underlying database. -# #user = await cfg.STORAGE.get_user({'id': user_id}) -# if not user: -# session = await aiohttp_session.get_session(request) -# del session['user'] -# return user - - -# async def _get_cur_user_id(request): -# session = await aiohttp_session.get_session(request) -# user = session.get(cfg['SESSION_USER_KEY']) -# while user: -# if not isinstance(user, dict): -# log.error('Wrong type of user_id in session') -# break - -# # Get a user ID from the user object. For now, we don't have a user database, so the session user is the same -# # as the "database-loaded" object user. Uncomment when we have a database. -# # user_id = cfg.STORAGE.user_id_from_string(user.user_id) -# # if not user_id: -# # break -# return user - -# if cfg['SESSION_USER_KEY'] in session: -# del session['user'] +# def login_required(handler): +# """ +# A handler function decorator that enforces that the user is logged in. If not, redirects to the login page. +# :param handler: function to decorate. +# :return: decorated function +# """ +# @user_to_request +# @wraps(handler) +# async def decorator(*args): +# request = _get_request(args) +# if not request[cfg.REQUEST_USER_KEY]: +# return _redirect(_get_login_url(request)) +# return await handler(*args) +# return decorator -# def _url_for(url_name, *args, **kwargs): -# if str(url_name).startswith(('/', 'http://', 'https://')): -# return url_name -# return cfg["APP"].router[url_name].url_for(*args, **kwargs) +# @user_to_request +# @template('index.html') +# async def index(request): +# """Web app home page.""" +# return { +# 'auth': {'cfg': cfg}, +# 'cur_user': request['user'], +# 'url_for': _url_for, +# } -# def _redirect(urlname, *args, **kwargs): -# return aiohttp.web.HTTPFound(_url_for(urlname, *args, **kwargs)) +# @login_required +# @template('users.html') +# async def users(request): +# """Handles an example private page that requires logging in.""" +# return {} -# def _get_request(args): -# # Supports class based views see web.View -# if isinstance(args[0], AbstractView): -# return args[0].request -# return args[-1] From c9338b4bd3eccc22ab1ff330592a704b00ced69f Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Fri, 7 Aug 2020 02:12:07 +0000 Subject: [PATCH 100/750] Moved auth into middleware --- webapp/aio_webapp_w.py | 2 ++ webapp/auth_handlers.py | 24 ++++++++++-------------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/webapp/aio_webapp_w.py b/webapp/aio_webapp_w.py index c392f14c4..68fb5c5b3 100644 --- a/webapp/aio_webapp_w.py +++ b/webapp/aio_webapp_w.py @@ -129,4 +129,6 @@ def fernet_key(s): fernet_key(settings.settings['aio']['session_secret']), max_age=settings.settings['aio']['session_max_age'])) +app.middlewares.append(auth_handlers.auth_middleware) + aiohttp.web.run_app(app, port=8888) diff --git a/webapp/auth_handlers.py b/webapp/auth_handlers.py index fed270f50..4f7589988 100644 --- a/webapp/auth_handlers.py +++ b/webapp/auth_handlers.py @@ -75,23 +75,19 @@ async def logout(request): return aiohttp.web.HTTPFound("/") ## TODO: Make a proper logout page +@aiohttp.web.middleware +async def auth_middleware(request, handler): + ''' + Move user into the request -def user_to_request(handler): - """ - A handler function decorator that adds user to request if user logged in. - :param handler: function to decorate. - :return: decorated function - """ - @wraps(handler) - async def decorator(*args): - request = args[0] - session = await aiohttp_session.get_session(request) - request['user'] = session.get('user', None) - return await handler(*args) - return decorator + Save user into a cookie + ''' + session = await aiohttp_session.get_session(request) + request['user'] = session.get('user', None) + resp = await handler(request) + return resp -@user_to_request async def user_info(request): return aiohttp.web.json_response(request['user']) From 551587a6ce01f407b30023069612b922982f9003 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Fri, 7 Aug 2020 19:44:35 +0000 Subject: [PATCH 101/750] Auth middleware, no decorator --- webapp/aio_webapp_w.py | 3 --- webapp/auth_handlers.py | 39 ++++++++++++++++++++------------------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/webapp/aio_webapp_w.py b/webapp/aio_webapp_w.py index 68fb5c5b3..f2aa47ae1 100644 --- a/webapp/aio_webapp_w.py +++ b/webapp/aio_webapp_w.py @@ -43,10 +43,7 @@ def static_file_handler(filename): ''' Serve a single static file ''' - @auth_handlers.user_to_request async def handler(request): - foo = await aiohttp_session.get_session(request) - print("Foo:", foo.get("user", {})) return aiohttp.web.FileResponse(filename) return handler diff --git a/webapp/auth_handlers.py b/webapp/auth_handlers.py index 4f7589988..c243e193e 100644 --- a/webapp/auth_handlers.py +++ b/webapp/auth_handlers.py @@ -35,6 +35,9 @@ from functools import wraps from yarl import URL +import json +import base64 + async def social(request): """Handles Google sign in. @@ -85,6 +88,23 @@ async def auth_middleware(request, handler): session = await aiohttp_session.get_session(request) request['user'] = session.get('user', None) resp = await handler(request) + if request['user'] is None: + userinfo = None + else: + userinfo = { + "name": request['user']['name'], + "picture": request['user']['picture'] + } + # This is a dumb way to sanitize data and pass to JS. + # + # Cookies tend to get encoded and decoded in ad-hoc strings a lot, often + # in non-compliant ways (to see why, try to find the spec for cookies!) + # + # This avoids bugs (and, should the issue come up, injections) + # + # This should really be abstracted away into a library which passes state + # back-and-forth. + resp.set_cookie("userinfo", base64.b64encode(json.dumps(userinfo).encode('utf-8')).decode('utf-8')) return resp @@ -163,22 +183,3 @@ async def _google(request): # return await handler(*args) # return decorator - -# @user_to_request -# @template('index.html') -# async def index(request): -# """Web app home page.""" -# return { -# 'auth': {'cfg': cfg}, -# 'cur_user': request['user'], -# 'url_for': _url_for, -# } - - -# @login_required -# @template('users.html') -# async def users(request): -# """Handles an example private page that requires logging in.""" -# return {} - - From cf91c95452f72ced94727b2d33587773ef9ea0bd Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Tue, 18 Aug 2020 17:36:57 +0000 Subject: [PATCH 102/750] Teachers template, better startup logic --- webapp/init.py | 52 +++++++++++++++++++++++ webapp/static_data/teachers.yaml.template | 19 +++++++++ 2 files changed, 71 insertions(+) create mode 100644 webapp/static_data/teachers.yaml.template diff --git a/webapp/init.py b/webapp/init.py index 87c1bdf30..c38d6260e 100644 --- a/webapp/init.py +++ b/webapp/init.py @@ -1,8 +1,23 @@ +''' +This file mostly confirms we have prerequisites for the system to work. + +We create a logs directory, grab 3rd party libraries, etc. +''' + +import hashlib import os +import shutil import sys if not os.path.exists("logs"): os.mkdir("logs") + print("Made logs directory") + + +if not os.path.exists("static_data/teachers.yaml"): + shutil.copyfile("static_data/teachers.yaml.template", "static_data/teachers.yaml") + print("Created a blank teachers file: static_data/teachers.yaml\n" + "Populate it with teacher accounts.") if not os.path.exists("../creds.yaml"): print(""" @@ -13,3 +28,40 @@ Fill in the missing fields. """) sys.exit(-1) + +if not os.path.exists("static/3rd_party"): + os.mkdir("static/3rd_party") + for name, url, sha in [ + ("require.js", "https://requirejs.org/docs/release/2.3.6/comments/require.js", "d1e7687c1b2990966131bc25a761f03d6de83115512c9ce85d72e4b9819fb8733463fa0d93ca31e2e42ebee6e425d811e3420a788a0fc95f745aa349e3b01901"), + ("bulma.min.css", "https://cdnjs.cloudflare.com/ajax/libs/bulma/0.9.0/css/bulma.min.css", "ec7342883fdb6fbd4db80d7b44938951c3903d2132fc3e4bf7363c6e6dc5295a478c930856177ac6257c32e1d1e10a4132c6c51d194b3174dc670ab8d116b362"), + ("fontawesome.js", "https://use.fontawesome.com/releases/v5.3.1/js/all.js -O fontawesome.js", "83e7b36f1545d5abe63bea9cd3505596998aea272dd05dee624b9a2c72f9662618d4bff6e51fafa25d41cb59bd97f3ebd72fd94ebd09a52c17c4c23fdca3962b"), + ("showdown.js", "https://rawgit.com/showdownjs/showdown/1.9.1/dist/showdown.js", "4fe14f17c2a1d0275d44e06d7e68d2b177779196c6d0c562d082eb5435eec4e710a625be524767aef3d9a1f6a5b88f912ddd71821f4a9df12ff7dd66d6fbb3c9"), + ("mustache.min.js", "http://cdnjs.cloudflare.com/ajax/libs/mustache.js/3.1.0/mustache.min.js", "e7c446dc9ac2da9396cf401774efd9bd063d25920343eaed7bee9ad878840e846d48204d62755aede6f51ae6f169dcc9455f45c1b86ba1b42980ccf8f241af25"), + ("d3.v5.min.js", "https://d3js.org/d3.v5.min.js", "466fe57816d719048885357cccc91a082d8e5d3796f227f88a988bf36a5c2ceb7a4d25842f5f3c327a0151d682e648cd9623bfdcc7a18a70ac05cfd0ec434463"), + ]: + filename = "static/3rd_party/{name}".format(name=name) + os.system("wget {url} -O {filename} 2> /dev/null".format( + url=url, + filename=filename + )) + shahash = hashlib.sha3_512(open(filename, "rb").read()).hexdigest() + print("Downloaded {name}".format(name=name)) + if shahash == sha: + print("File integrity confirmed!") + else: + print("Incorrect SHA hash. Something odd is going on. DO NOT IGNORE THIS ERROR/WARNING") + print() + print("Expected SHA: " + sha) + print("Actual SHA: " + shahash) + print() + print("We download 3rd party libraries from the Internet. This error means that ones of") + print("these files changed. This may indicate a man-in-the-middle attack, that a CDN has") + print("been compromised, or more prosaically, that one of the files had something like") + print("a security fix backported. In either way, VERIFY what happened before moving on.") + print("If unsure, please consult with a security expert.") + print() + print("This error should never happen unless someone is under attack (or there is a") + print("serious bug).") + os.exit(-1) + + print() diff --git a/webapp/static_data/teachers.yaml.template b/webapp/static_data/teachers.yaml.template new file mode 100644 index 000000000..15fa3dd3b --- /dev/null +++ b/webapp/static_data/teachers.yaml.template @@ -0,0 +1,19 @@ +# This is a template YAML file of authorized teachers. +# +# Each line contains one teacher: +# +# { +# "vsweeny@polk.apsd.us": {"google_id": "1234567890"}, +# "mfornz@polk.apsd.us": {"google_id": "1234567891"}, +# "jlee@polk.apsd.us": {"google_id": "1234567891"} +# } +# +# This should be the official Google Classroom email address, and the +# Google ID of the teacher. +# +# The first time this application is run, this template will be used +# to create a teachers.yaml file. You should not edit this one, but +# that one. + +{ +} From 16dd75a25149601efcca9f73606a7492528895d4 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Tue, 18 Aug 2020 17:37:22 +0000 Subject: [PATCH 103/750] Ignore commiting data file --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 988ebdfcc..c0904debe 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ .\#* *__pycache__* webapp/logs -creds.yaml \ No newline at end of file +webapp/static_data/teachers.yaml +creds.yaml From 0bdd6e45f03d6a3da9f717c1fa0c16f28e8112c0 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sat, 22 Aug 2020 12:05:06 +0000 Subject: [PATCH 104/750] Support for Google Classroom, class rosters, front-end rendering, etc. Also, adding scripts to generate test data. Also, cleaning up initialization scripts --- webapp/aio_webapp_w.py | 19 ++- webapp/auth_handlers.py | 88 +++++++++--- webapp/init.py | 4 +- webapp/rosters.py | 87 +++++++++++ webapp/static/index.html | 2 +- webapp/static/modules/course.html | 45 ++++++ webapp/static/modules/courses.html | 21 +++ webapp/static/modules/login.html | 3 + webapp/static/modules/navbar_loggedin.html | 17 +++ webapp/static/webapp.html | 55 +++++++ webapp/static/webapp.js | 136 ++++++++++++++++++ webapp/static_data/README.md | 31 ++++ .../static_data/make_dummy_test_user_tsv.py | 26 ++++ .../make_google_classroom_test_courses.py | 78 ++++++++++ 14 files changed, 587 insertions(+), 25 deletions(-) create mode 100644 webapp/rosters.py create mode 100644 webapp/static/modules/course.html create mode 100644 webapp/static/modules/courses.html create mode 100644 webapp/static/modules/login.html create mode 100644 webapp/static/modules/navbar_loggedin.html create mode 100644 webapp/static/webapp.html create mode 100644 webapp/static/webapp.js create mode 100644 webapp/static_data/README.md create mode 100644 webapp/static_data/make_dummy_test_user_tsv.py create mode 100644 webapp/static_data/make_google_classroom_test_courses.py diff --git a/webapp/aio_webapp_w.py b/webapp/aio_webapp_w.py index f2aa47ae1..adafc6186 100644 --- a/webapp/aio_webapp_w.py +++ b/webapp/aio_webapp_w.py @@ -22,6 +22,7 @@ import event_pipeline import student_data import auth_handlers +import rosters import settings @@ -48,6 +49,17 @@ async def handler(request): return handler +async def index(request): + print(request['user']) + print(type(request['user'])) + if request['user'] is None: + print("Index") + return aiohttp.web.FileResponse("static/index.html") + else: + print("Course list") + return aiohttp.web.FileResponse("static/courselist.html") + + def static_directory_handler(basepath): ''' Serve static files from a directory. @@ -84,6 +96,8 @@ def handler(request): # Serve static files app.add_routes([ aiohttp.web.get('/static/{filename}', static_directory_handler("static")), + aiohttp.web.get('/static/modules/{filename}', static_directory_handler("static/modules")), + aiohttp.web.get('/static/3rd_party/{filename}', static_directory_handler("static/3rd_party")), aiohttp.web.get('/static/media/{filename}', static_directory_handler("media")), aiohttp.web.get('/static/media/avatar/{filename}', static_directory_handler("media/hubspot_persona_images/")), @@ -99,11 +113,14 @@ def handler(request): app.add_routes([ aiohttp.web.get('/webapi/event/', event_pipeline.ajax_event_request), aiohttp.web.post('/webapi/event/', event_pipeline.ajax_event_request), + aiohttp.web.get('/webapi/courselist/', rosters.courselist_api), + aiohttp.web.get('/webapi/courseroster/{course_id}', rosters.courseroster_api), ]) # Generic web-appy things app.add_routes([ - aiohttp.web.get('/', static_file_handler("static/index.html")), +# aiohttp.web.get('/', static_file_handler("static/index.html")), + aiohttp.web.get('/', index), aiohttp.web.get('/auth/login/{provider:google}', handler=auth_handlers.social), aiohttp.web.get('/auth/logout', handler=auth_handlers.logout), aiohttp.web.get('/auth/userinfo', handler=auth_handlers.user_info) diff --git a/webapp/auth_handlers.py b/webapp/auth_handlers.py index c243e193e..cb375ebde 100644 --- a/webapp/auth_handlers.py +++ b/webapp/auth_handlers.py @@ -36,8 +36,29 @@ from yarl import URL import json +import yaml import base64 +import rosters + + +async def verify_teacher_account(email, google_id): + ''' + Confirm the teacher is registered with the system. Eventually, we will want + 3 versions of this: + * Always true (open system) + * Text file backed (pilots, small deploys) + * Database-backed (large-scale deploys) + + For now, we have the file-backed version + ''' + users = yaml.safe_load(open("static_data/teachers.yaml")) + if email not in users: + return False + if users[email]["google_id"] != google_id: + return False + return True + async def social(request): """Handles Google sign in. @@ -52,9 +73,12 @@ async def social(request): if 'user_id' in user: # User ID returned in 'data', authorize user. - await _authorize_user(request, user) - url = user['back_to'] or "/" - return aiohttp.web.HTTPFound(url) + authorized = await _authorize_user(request, user) + if authorized: + url = user['back_to'] or "/" + return aiohttp.web.HTTPFound(url) + else: + url = "/static/unauth.html" ## Login failed. TODO: Make a proper login failed page. return aiohttp.web.HTTPFound("/") @@ -68,13 +92,17 @@ async def _authorize_user(request, user): """ session = await aiohttp_session.get_session(request) session["user"] = user + return verify_teacher_account(user['user_id'], user['email']) async def logout(request): """Handles sign out. This is generic - does not depend on which social ID is logged in - (Google/Facebook/...).""" + (Google/Facebook/...). + """ session = await aiohttp_session.get_session(request) session.pop("user", None) + session.pop("auth_headers", None) + print(session) return aiohttp.web.HTTPFound("/") ## TODO: Make a proper logout page @@ -87,6 +115,7 @@ async def auth_middleware(request, handler): ''' session = await aiohttp_session.get_session(request) request['user'] = session.get('user', None) + request['auth_headers'] = session.get('auth_headers', None) resp = await handler(request) if request['user'] is None: userinfo = None @@ -95,15 +124,16 @@ async def auth_middleware(request, handler): "name": request['user']['name'], "picture": request['user']['picture'] } - # This is a dumb way to sanitize data and pass to JS. + # This is a dumb way to sanitize data and pass it to the front-end. # # Cookies tend to get encoded and decoded in ad-hoc strings a lot, often # in non-compliant ways (to see why, try to find the spec for cookies!) # - # This avoids bugs (and, should the issue come up, injections) + # This avoids bugs (and, should the issue come up, security issues + # like injections) # # This should really be abstracted away into a library which passes state - # back-and-forth. + # back-and-forth, but for now, this works. resp.set_cookie("userinfo", base64.b64encode(json.dumps(userinfo).encode('utf-8')).decode('utf-8')) return resp @@ -131,7 +161,9 @@ async def _google(request): params.update({ 'response_type': 'code', 'scope': ('https://www.googleapis.com/auth/userinfo.profile' - ' https://www.googleapis.com/auth/userinfo.email'), + ' https://www.googleapis.com/auth/userinfo.email' + ' https://www.googleapis.com/auth/classroom.courses.readonly' + ' https://www.googleapis.com/auth/classroom.rosters.readonly'), }) if 'back_to' in request.query: params['state'] = request.query[back_to] @@ -153,10 +185,20 @@ async def _google(request): # get user profile headers = {'Authorization': 'Bearer ' + data['access_token']} + session = await aiohttp_session.get_session(request) + session["auth_headers"] = headers + request["auth_headers"] = headers + # Old G+ URL that's no longer supported. url = 'https://www.googleapis.com/oauth2/v1/userinfo' async with client.get(url, headers=headers) as resp: profile = await resp.json() + print(profile) + for course in (await rosters.courselist(request)): + print(json.dumps(course, indent=3)) + print(await rosters.courseroster(request, course['id'])) + #async with client.get('"), headers=headers) as resp: + # print(await resp.text()) return { 'user_id': profile['id'], @@ -168,18 +210,20 @@ async def _google(request): } -# def login_required(handler): -# """ -# A handler function decorator that enforces that the user is logged in. If not, redirects to the login page. -# :param handler: function to decorate. -# :return: decorated function -# """ -# @user_to_request -# @wraps(handler) -# async def decorator(*args): -# request = _get_request(args) -# if not request[cfg.REQUEST_USER_KEY]: -# return _redirect(_get_login_url(request)) -# return await handler(*args) -# return decorator +def html_login_required(handler): + """ + A handler function decorator that enforces that the user is logged + in. If not, redirects to the login page. + + :param handler: function to decorate. + :return: decorated function + + """ + @wraps(handler) + async def decorator(*args): + user = args[0]["user"] + if user is None: + return aiohttp.web.HTTPFound("/") + return handler(*args) + return decorator diff --git a/webapp/init.py b/webapp/init.py index c38d6260e..6f37278c5 100644 --- a/webapp/init.py +++ b/webapp/init.py @@ -33,9 +33,11 @@ os.mkdir("static/3rd_party") for name, url, sha in [ ("require.js", "https://requirejs.org/docs/release/2.3.6/comments/require.js", "d1e7687c1b2990966131bc25a761f03d6de83115512c9ce85d72e4b9819fb8733463fa0d93ca31e2e42ebee6e425d811e3420a788a0fc95f745aa349e3b01901"), + ("text.js", "https://raw.githubusercontent.com/requirejs/text/3f9d4c19b3a1a3c6f35650c5788cbea1db93197a/text.js", "fb8974f1633f261f77220329c7070ff214241ebd33a1434f2738572608efc8eb6699961734285e9500bbbd60990794883981fb113319503208822e6706bca0b8"), ("bulma.min.css", "https://cdnjs.cloudflare.com/ajax/libs/bulma/0.9.0/css/bulma.min.css", "ec7342883fdb6fbd4db80d7b44938951c3903d2132fc3e4bf7363c6e6dc5295a478c930856177ac6257c32e1d1e10a4132c6c51d194b3174dc670ab8d116b362"), ("fontawesome.js", "https://use.fontawesome.com/releases/v5.3.1/js/all.js -O fontawesome.js", "83e7b36f1545d5abe63bea9cd3505596998aea272dd05dee624b9a2c72f9662618d4bff6e51fafa25d41cb59bd97f3ebd72fd94ebd09a52c17c4c23fdca3962b"), ("showdown.js", "https://rawgit.com/showdownjs/showdown/1.9.1/dist/showdown.js", "4fe14f17c2a1d0275d44e06d7e68d2b177779196c6d0c562d082eb5435eec4e710a625be524767aef3d9a1f6a5b88f912ddd71821f4a9df12ff7dd66d6fbb3c9"), + ("showdown.js.map", "https://rawgit.com/showdownjs/showdown/1.9.1/dist/showdown.js.map", "74690aa3cea07fd075942ba9e98cf7297752994b93930acb3a1baa2d3042a62b5523d3da83177f63e6c02fe2a09c8414af9e1774dad892a303e15a86dbeb29ba"), ("mustache.min.js", "http://cdnjs.cloudflare.com/ajax/libs/mustache.js/3.1.0/mustache.min.js", "e7c446dc9ac2da9396cf401774efd9bd063d25920343eaed7bee9ad878840e846d48204d62755aede6f51ae6f169dcc9455f45c1b86ba1b42980ccf8f241af25"), ("d3.v5.min.js", "https://d3js.org/d3.v5.min.js", "466fe57816d719048885357cccc91a082d8e5d3796f227f88a988bf36a5c2ceb7a4d25842f5f3c327a0151d682e648cd9623bfdcc7a18a70ac05cfd0ec434463"), ]: @@ -62,6 +64,6 @@ print() print("This error should never happen unless someone is under attack (or there is a") print("serious bug).") - os.exit(-1) + sys.exit(-1) print() diff --git a/webapp/rosters.py b/webapp/rosters.py new file mode 100644 index 000000000..0633b825b --- /dev/null +++ b/webapp/rosters.py @@ -0,0 +1,87 @@ +import aiohttp +import aiohttp.web + +import settings + +COURSE_URL = 'https://classroom.googleapis.com/v1/courses' +ROSTER_URL = 'https://classroom.googleapis.com/v1/courses/{courseid}/students' + +def clean_data(resp_json, key, sort_key, default=None): + print("Response", resp_json) + if 'error' in resp_json: + return {'error': resp_json['error']} # Typically, resp_json['error'] == 'UNAUTHENTICATED' + if key is not None: + if key in resp_json: + resp_json = resp_json[key] + # This happens if e.g. no courses. Google seems to just return {} + # instead of {'courses': []} + else: + return default + if sort_key is not None: + resp_json.sort(key=sort_key) + print(resp_json) + return resp_json + + +async def synthetic_ajax(request, url, key=None, sort_key=None, default=None): + ''' + Stub similar to google_ajax, but grabbing data from local files. + + This is helpful for testing, but it's even more helpful since + Google is an amazingly unreliable B2B company, and this lets us + develop without relying on them. + ''' + synthetic_data = { + COURSE_URL: "static_data/courses.json", + ROSTER_URL: "static_data/students.json" + } + return clean_data(open(synthetic_data[url]).read(), default=default) + +async def google_ajax(request, url, parameters={}, key=None, sort_key=None, default=None): + ''' + Request information through Google's API + + Most requests return a dictionary with one key. If we just want + that element, set `key` to be the element of the dictionary we want + + This is usually a list. If we want to sort this, pass a function as + `sort_key` + + Note that we return error as a json object with error information, + rather than raising an exception. In most cases, we want to pass + this error back to the JavaScript client, which can then handle + loading the auth page. + ''' + async with aiohttp.ClientSession(loop=request.app.loop) as client: + async with client.get(url.format(**parameters), headers=request["auth_headers"]) as resp: + resp_json = await resp.json() + return clean_data(resp_json, key, sort_key, default=default) + +async def courselist(request): + course_list = await google_ajax( + request, + url=COURSE_URL, + key='courses', + sort_key=lambda x:x.get('name', 'ZZ'), + default=[] + ) + return course_list + + +async def courseroster(request, course_id): + roster = await google_ajax( + request, + url=ROSTER_URL, + parameters={'courseid': int(course_id)}, + key='students', + sort_key=lambda x:x.get('name', {}).get('fullName', 'ZZ'), + default=[] + ) + return roster + +async def courselist_api(request): + return aiohttp.web.json_response(await courselist(request)) + +async def courseroster_api(request): + course_id = int(request.match_info['course_id']) + return aiohttp.web.json_response(await courseroster(request, course_id)) diff --git a/webapp/static/index.html b/webapp/static/index.html index d671f59c6..f3c3132d8 100644 --- a/webapp/static/index.html +++ b/webapp/static/index.html @@ -6,7 +6,7 @@ - + diff --git a/webapp/static/modules/course.html b/webapp/static/modules/course.html new file mode 100644 index 000000000..50f6fa1d9 --- /dev/null +++ b/webapp/static/modules/course.html @@ -0,0 +1,45 @@ +
+
+
+ {{ name }} +
+
+
+
+

+

+ {{ descriptionHeading }}

+
+
+
+
+

+

+

+
+
+
+ + + diff --git a/webapp/static/modules/courses.html b/webapp/static/modules/courses.html new file mode 100644 index 000000000..7f183437a --- /dev/null +++ b/webapp/static/modules/courses.html @@ -0,0 +1,21 @@ +
+

My Courses

+
+ +
+
diff --git a/webapp/static/modules/login.html b/webapp/static/modules/login.html new file mode 100644 index 000000000..282348a9f --- /dev/null +++ b/webapp/static/modules/login.html @@ -0,0 +1,3 @@ + + + diff --git a/webapp/static/modules/navbar_loggedin.html b/webapp/static/modules/navbar_loggedin.html new file mode 100644 index 000000000..949c80fe4 --- /dev/null +++ b/webapp/static/modules/navbar_loggedin.html @@ -0,0 +1,17 @@ +
+
+ {{ user_name }} +
+
+
+ +
+
+
+
+ + Log out + +
+
+
diff --git a/webapp/static/webapp.html b/webapp/static/webapp.html new file mode 100644 index 000000000..0d750c979 --- /dev/null +++ b/webapp/static/webapp.html @@ -0,0 +1,55 @@ + + + + + + + + + + + Writing Analysis + + + +
+ + + + +
+ +
+
+
+

+ Writing Dashboard + by Piotr Mitros. Copyright + (c) 2020. Educational Testing + Service. The source + code will be released as free / open source software, + most likely + under the + AGPLv3 license. Some thoughts on privacy. +

+
+
+
+ + diff --git a/webapp/static/webapp.js b/webapp/static/webapp.js new file mode 100644 index 000000000..b72cbe90d --- /dev/null +++ b/webapp/static/webapp.js @@ -0,0 +1,136 @@ +function getCookie(cname) { + var name = cname + "="; + var decodedCookie = decodeURIComponent(document.cookie); + var ca = decodedCookie.split(';'); + for(var i = 0; i Date: Sun, 23 Aug 2020 01:13:07 +0000 Subject: [PATCH 105/750] Filesystem startup snapshot function --- webapp/filesystem_state.py | 55 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 webapp/filesystem_state.py diff --git a/webapp/filesystem_state.py b/webapp/filesystem_state.py new file mode 100644 index 000000000..4fd34fc1b --- /dev/null +++ b/webapp/filesystem_state.py @@ -0,0 +1,55 @@ +import hashlib +import os +import subprocess + +extensions = [ + ".py", + ".js", + ".html", + ".md" +] + +filestring = """{filename}: +\thash:{hash} +\tst_mode:{st_mode} +\tst_size:{st_size} +\tst_atime:{st_atime} +\tst_mtime:{st_mtime} +\tst_ctime:{st_ctime} +""" + +def filesystem_state(): + ''' + Make a snapshot of the file system. Return a json object. Best + usage is to combine with `yaml.dump`, or `json.dump` with a + specific indent. This is helpful for knowing which version was running. + + Snapshot contains list of Python, HTML, JSON, and Markdown files, + together with their SHA hashes and modified times. It also + contains a `git` hash of the current commit. + + This ought to be enough to confirm which version of the tool is + running, and if we are running from a `git` commit (as we ought to + in production) or if changes were made since git commited. + ''' + file_info = {} + for root, dirs, files in os.walk("."): + for name in files: + for extension in extensions: + if name.endswith(extension): + filename = os.path.join(root, name) + stat = os.stat(filename) + file_info[filename] = { + "hash": hashlib.sha3_512(open(filename, "rb").read()).hexdigest(), + "st_mode": stat.st_mode, + "st_size": stat.st_size, + "st_atime": stat.st_atime, + "st_mtime": stat.st_mtime, + "st_ctime": stat.st_ctime + } + file_info['::git-head::'] = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('utf-8').strip() + return file_info + +if __name__ == '__main__': + import yaml + print(yaml.dump(filesystem_state())) From a819b23447a779b388e822753bddd1af50bbbfa7 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 23 Aug 2020 01:13:40 +0000 Subject: [PATCH 106/750] Moving to main webapp on /, as well as adding docs --- webapp/aio_webapp_w.py | 4 +-- webapp/log_event.py | 48 +++++++++++++++++++++++++++++++ webapp/static/modules/course.html | 2 +- webapp/static/modules/login.html | 2 +- webapp/static/webapp.html | 4 +-- webapp/static/webapp.js | 42 ++++++++++++++++++--------- 6 files changed, 83 insertions(+), 19 deletions(-) diff --git a/webapp/aio_webapp_w.py b/webapp/aio_webapp_w.py index adafc6186..d0fe80f56 100644 --- a/webapp/aio_webapp_w.py +++ b/webapp/aio_webapp_w.py @@ -119,8 +119,8 @@ def handler(request): # Generic web-appy things app.add_routes([ -# aiohttp.web.get('/', static_file_handler("static/index.html")), - aiohttp.web.get('/', index), + aiohttp.web.get('/', static_file_handler("static/webapp.html")), +# aiohttp.web.get('/', index), aiohttp.web.get('/auth/login/{provider:google}', handler=auth_handlers.social), aiohttp.web.get('/auth/logout', handler=auth_handlers.logout), aiohttp.web.get('/auth/userinfo', handler=auth_handlers.user_info) diff --git a/webapp/log_event.py b/webapp/log_event.py index 9dc4aa237..2f46522d3 100644 --- a/webapp/log_event.py +++ b/webapp/log_event.py @@ -1,3 +1,51 @@ +# For now, we dump logs into files, crudely. +# +# We're not there yet, but we would like to create a 哈希树, or +# Merkle-tree-style structure for our log files. +# +# Or to be specific, a Merkle DAG, like git. +# +# Each item is stored under its SHA hash. Note that items are not +# guaranteed to exist. We can prune them, and leave a dangling pointer +# with just the SHA. +# +# Each event log will be structured as +# +-----------------+ +-----------------+ +# <--- Last item (SHA) | <--- Last item (SHA) | ... +# | | | | +# | Data (SHA) | | Data (SHA) | +# +-------|---------+ +--------|--------+ +# | | +# v v +# +-------+ +-------+ +# | Event | | Event | +# +-------+ +-------+ +# +# Where the top objects form a linked list (each containing a pair of +# SHA hashes, one of the previous item, and one of the associated +# event). +# +# We will then have a hierarchy, where we have lists per-document, +# documents per-student. When we run analyses, those will store the +# hashes of where in each event log we are. Likewise, with each layer +# of analysis, we'll store pointers to git hashes of code, as well as +# of intermediate files (and how those were generated). +# +# Where data is available, we can confirm we're correctly replicating +# prior tesults. +# +# The planned data structure is very similar to git, but with the +# potential for missing data without an implosion. +# +# Where data might not be available is after a FERPA, CCPA, or GDPR +# requests to change data. In those cases, we'll have dangling nodes, +# where we'll know that data used to exist, but not what it was. +# +# We might also have missing intermediate files. For example, if we do +# a dozen analyses, we'll want to know those happened and what those +# were, but we might not keep terabytes of data around (just enough to +# redo those analyses). + import datetime import inspect import json diff --git a/webapp/static/modules/course.html b/webapp/static/modules/course.html index 50f6fa1d9..6b12a88e6 100644 --- a/webapp/static/modules/course.html +++ b/webapp/static/modules/course.html @@ -6,7 +6,7 @@
-

diff --git a/webapp/static/modules/login.html b/webapp/static/modules/login.html index 282348a9f..6c8fd32f9 100644 --- a/webapp/static/modules/login.html +++ b/webapp/static/modules/login.html @@ -1,3 +1,3 @@ - + diff --git a/webapp/static/webapp.html b/webapp/static/webapp.html index 0d750c979..cd638df65 100644 --- a/webapp/static/webapp.html +++ b/webapp/static/webapp.html @@ -5,7 +5,7 @@ - + Writing Analysis @@ -17,7 +17,7 @@