diff --git a/.gitignore b/.gitignore index 6190589..f1eb964 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ web/public/splunkbot/map.json node_modules .DS_Store *.DS_Store +cron.sh +web/public/splunkbot/map_*.json diff --git a/genmapjson.js b/genmapjson.js index bed2f3f..637a3ad 100644 --- a/genmapjson.js +++ b/genmapjson.js @@ -70,7 +70,11 @@ function writeResults(results, channel) { } -for (var i=0; i < WEBCONFIG.channels.length; i++) { +var doit = function(i) { + if (i >= WEBCONFIG.channels.length) { + return; + } + var channel = WEBCONFIG.channels[i]; var searchstring = 'search `irclogs` | search [ search `irclogs` | search to='+channel+' | ' +'stats count by nick | sort 15 -count | fields nick | format ] | ' @@ -82,8 +86,13 @@ for (var i=0; i < WEBCONFIG.channels.length; i++) { // console.log("Search: "+searchstring); splunk.search(searchstring, function(err, results) { - writeResults(results, channel); + if (!(typeof results.messages[0] == 'object' && results.messages[0].type == 'FATAL')) { + writeResults(results, channel); + } + doit(i+1); }, "-7d", "now"); -} +}; + +doit(0); -// writeResults(testresults); \ No newline at end of file +// writeResults(testresults); diff --git a/node_modules/.bin/jade b/node_modules/.bin/jade index 571fae7..65a3bac 120000 --- a/node_modules/.bin/jade +++ b/node_modules/.bin/jade @@ -1 +1 @@ -../jade/bin/jade \ No newline at end of file +../jade/bin/jade.js \ No newline at end of file diff --git a/package.json b/package.json index d5e9da8..6941af8 100644 --- a/package.json +++ b/package.json @@ -1,37 +1,42 @@ { - "name" : "splunkbot", - "version" : "0.1.0", - "description" : "An IRC Bot which logs traffic from configured channels to Splunk and provides a web interface for easily searching and extracting data from IRC", - "homepage": "http://github.com/coccyx/Splunkbot", - "main" : "index.js", - "directories" : { - "example" : "examples", - "lib" : "lib", - "test" : "tests" - }, - "repository" : { - "type" : "git", - "url" : "http://github.com/coccyx/Splunkbot.git" - }, - "keywords" : [ - "splunk", - "irc" - ], - "dependencies" : { - "splunk-sdk" : ">= 0.5.0", - "config" : ">= 0.4.10", - "irc": ">= 0.3.3", - "request": "2.2.x", - "loggly": ">= 0.3.11", - "express": ">= 2.5.8", - "less": ">= 0.0.1", - "jade": ">= 0.0.1" - }, - "author" : { - "name" : "Clint Sharp", - "email" : "clint@typhoon.org", - "url" : "http://clintsharp.com/" - }, - "license" : "Apache 2.0", - "engine" : { "node" : ">=0.4.9" } + "name": "splunkbot", + "version": "0.1.0", + "description": "An IRC Bot which logs traffic from configured channels to Splunk and provides a web interface for easily searching and extracting data from IRC", + "homepage": "http://github.com/coccyx/Splunkbot", + "main": "index.js", + "directories": { + "example": "examples", + "lib": "lib", + "test": "tests" + }, + "repository": { + "type": "git", + "url": "http://github.com/coccyx/Splunkbot.git" + }, + "keywords": [ + "splunk", + "irc" + ], + "dependencies": { + "body-parser": "^1.9.1", + "config": ">= 0.4.10", + "errorhandler": "^1.2.2", + "express": ">= 2.5.8", + "irc": ">= 0.3.3", + "jade": ">= 0.0.1", + "less": ">= 0.0.1", + "loggly": ">= 0.3.11", + "method-override": "^2.3.0", + "request": "2.2.x", + "splunk-sdk": ">= 0.5.0" + }, + "author": { + "name": "Clint Sharp", + "email": "clint@typhoon.org", + "url": "http://clintsharp.com/" + }, + "license": "Apache 2.0", + "engine": { + "node": ">=0.4.9" + } } diff --git a/web/app.js b/web/app.js index 4213170..9682602 100644 --- a/web/app.js +++ b/web/app.js @@ -8,30 +8,24 @@ var express = require('express') , CONFIG = require('config').web , search = require('../search') , irclog = require('../irclog') + , bodyParser = require('body-parser') + , methodOverride = require('method-override') + , errorHandler = require('errorhandler') , pagevars = { 'serverTZOffset': CONFIG.server_tz_offset} , request = require('request'); -var app = module.exports = express.createServer(); +var app = module.exports = express(); // Configuration -app.configure(function(){ +//app.configure(function(){ app.set('views', __dirname + '/views'); app.set('view engine', 'jade'); - app.use(express.bodyParser()); - app.use(express.methodOverride()); - app.use(app.router); + app.use(bodyParser()); + app.use(methodOverride()); +// app.use(app.router); app.use(express.static(__dirname + '/public')); -}); - -app.configure('development', function(){ - app.use(express.errorHandler({ dumpExceptions: true, showStack: true })); - app.set('view options', { pretty: true }); -}); - -app.configure('production', function(){ - app.use(express.errorHandler()); -}); +//}); // Routes @@ -218,6 +212,23 @@ app.all('/proxy/*', function (req, res) { } }); +if (app.get('env') == 'development') { + +//app.configure('development', function(){ + app.use(errorHandler({ dumpExceptions: true, showStack: true })); + app.set('view options', { pretty: true }); +//}); + +} else { + +//app.configure('production', function(){ + app.use(errorHandler()); +//}); + +} -app.listen(CONFIG.port); -console.log("Express server listening on port %d in %s mode", app.address().port, app.settings.env); + + +app.listen(CONFIG.port, function() { + console.log("Express server listening on port %d in %s mode", app.get('port'), app.settings.env); +}); diff --git a/web/public/splunkbot/splunkbot.js b/web/public/splunkbot/splunkbot.js index bdb46ac..276afd6 100644 --- a/web/public/splunkbot/splunkbot.js +++ b/web/public/splunkbot/splunkbot.js @@ -724,48 +724,17 @@ Splunkbot.prototype.timeline = function(channel, timewindow) { } }; - Async.chain([ - // Login - function(callback) { splunkbot.service.login(callback); }, - // Create the job - function(success, callback) { - splunkbot.service.jobs().create(searchTerm, {status_buckets: 300}, callback); - }, - // Loop until the job is "done" - function(job, callback) { - var searcher = new splunkjs.JobManager(job.service, job); - - // Queue up timeline displays while we are querying the job - searcher.on("progress", function(properties) { - job.timeline({}, function(err, data) { - if (!err) updateTimeline(data); - }); + splunkbot.service.search(searchTerm, {status_buckets: 300}, function(err, job) { + job.track({period: 200}, { + progress: function(properties) { + job.timeline({}, function(err, data) { + if (!err) updateTimeline(data); + }); + }, + done: function(job) { + job.timeline({}); + } }); - - // Move forward once the search is done - searcher.on("done", callback); - }, - // Get the final timeline data - function(searcher, callback) { - searcher.job.timeline({}, callback); - }, - // Update the timeline control - function(timelineData, job, callback) { - updateTimeline(timelineData); - callback(null, job); - } - ], - // And we're done, so make sure we had no error, and - // cancel the job - function(err, job) { - if (err) { - console.log(err); - alert("An error occurred"); - } - - if (job) { - job.cancel(); - } }); } @@ -792,45 +761,16 @@ Splunkbot.prototype.toptalkers = function(channel, timewindow, chartToken) { var chart = new splunkjs.UI.Charting.Chart("#toptalkers", splunkjs.UI.Charting.ChartType.PIE, false); - Async.chain([ - // Login - function(callback) { splunkbot.service.login(callback); }, - // Create the job - function(success, callback) { - splunkbot.service.jobs().create(searchTerm, {status_buckets: 300}, callback); - }, - // Loop until the job is "done" - function(job, callback) { - var searcher = new splunkjs.JobManager(job.service, job); - - // Move forward once the search is done - searcher.on("done", callback); - }, - // Get the final results data - function(searcher, callback) { - searcher.job.results({output_mode: "json_cols"}, callback); - }, - // Update the chart - function(results, job, callback) { - splunkjs.UI.ready(chartToken, function() { - spinner2.spin(); - chart.setData(results, { }); - chart.draw(); - callback(null, job); + splunkbot.service.search(searchTerm, {status_buckets: 300}, function(err,job) { + job.track({period:200}, {done: function(job) { + job.results({output_mode: "json_cols"}, function(results) { + splunkjs.UI.ready(chartToken, function() { + spinner2.spin(); + chart.setData(results, { }); + chart.draw(); + }); }); - } - ], - // And we're done, so make sure we had no error, and - // cancel the job - function(err, job) { - if (err) { - console.log(err); - alert("An error occurred"); - } - - if (job) { - job.cancel(); - } + }}); }); } @@ -850,45 +790,16 @@ Splunkbot.prototype.mostmentioned = function(channel, timewindow, chartToken) { var chart = new splunkjs.UI.Charting.Chart("#mostmentioned", splunkjs.UI.Charting.ChartType.PIE, false); - Async.chain([ - // Login - function(callback) { splunkbot.service.login(callback); }, - // Create the job - function(success, callback) { - splunkbot.service.jobs().create(searchTerm, {status_buckets: 300}, callback); - }, - // Loop until the job is "done" - function(job, callback) { - var searcher = new splunkjs.JobManager(job.service, job); - - // Move forward once the search is done - searcher.on("done", callback); - }, - // Get the final results data - function(searcher, callback) { - searcher.job.results({output_mode: "json_cols"}, callback); - }, - // Update the chart - function(results, job, callback) { - splunkjs.UI.ready(chartToken, function() { - spinner3.spin(); - chart.setData(results, { }); - chart.draw(); - callback(null, job); + splunkbot.service.search(searchTerm, {status_buckets: 300}, function(err, job) { + job.track({period:200}, {done: function(job) { + job.results({output_mode: "json_cols"}, function(results) { + splunkjs.UI.ready(chartToken, function() { + spinner3.spin(); + chart.setData(results, { }); + chart.draw(); + }); }); - } - ], - // And we're done, so make sure we had no error, and - // cancel the job - function(err, job) { - if (err) { - console.log(err); - alert("An error occurred"); - } - - if (job) { - job.cancel(); - } + }}); }); } diff --git a/web/views/index.jade b/web/views/index.jade index 2945358..e42f7bf 100644 --- a/web/views/index.jade +++ b/web/views/index.jade @@ -1,41 +1,44 @@ -.hero-unit - h1 Welcome to Splunkbot! - p - | Splunkbot is an IRC bot written in node.js. Splunkbot logs IRC traffic to Splunk and provides - | a really nice web interface for searching and analyzing those logs. - p - | We can do really cool things with the data, including some cool visualizations like our - a(href='/map') Map! - p - a.btn.btn-success.btn-large(href='http://github.com/coccyx/Splunkbot') Click here for our Github project » -.row - .span3 - h2 Stats +extends layout + +block body + .hero-unit + h1 Welcome to Splunkbot! p - | Check out analytics of the IRC Channels Splunkbot is on (EFNet #splunk) - .span3 - h2 URLs + | Splunkbot is an IRC bot written in node.js. Splunkbot logs IRC traffic to Splunk and provides + | a really nice web interface for searching and analyzing those logs. + p + | We can do really cool things with the data, including some cool visualizations like our + a(href='/map') Map! p - | Before Twitter, link blogging, etc, there was IRC. Check out the last URLs posted to channels to see what we're linking to. - .span3 - h2 Live - p - | Using Splunk's real-time search capabilities, we can provide a realtime view into an IRC channel without being logged onto IRC, - | merely by looking at a real-time search of the logs. - .span3 - h2 Search - p - | Last but not least, use Splunk's superior searching technology to search past logs, by nickname or any other text. -.row - .span3 - p - a.btn.btn-success(href='/stats') Click here » - .span3 - p - a.btn.btn-success(href='/urls') Click here » - .span3 - p - a.btn.btn-success(href='/live') Click here » - .span3 - form(action='/search') - input(type='text', placeholder='Search', name='q').search-query.span2 \ No newline at end of file + a.btn.btn-success.btn-large(href='http://github.com/coccyx/Splunkbot') Click here for our Github project » + .row + .span3 + h2 Stats + p + | Check out analytics of the IRC Channels Splunkbot is on (EFNet #splunk) + .span3 + h2 URLs + p + | Before Twitter, link blogging, etc, there was IRC. Check out the last URLs posted to channels to see what we're linking to. + .span3 + h2 Live + p + | Using Splunk's real-time search capabilities, we can provide a realtime view into an IRC channel without being logged onto IRC, + | merely by looking at a real-time search of the logs. + .span3 + h2 Search + p + | Last but not least, use Splunk's superior searching technology to search past logs, by nickname or any other text. + .row + .span3 + p + a.btn.btn-success(href='/stats') Click here » + .span3 + p + a.btn.btn-success(href='/urls') Click here » + .span3 + p + a.btn.btn-success(href='/live') Click here » + .span3 + form(action='/search') + input(type='text', placeholder='Search', name='q').search-query.span2 diff --git a/web/views/layout.jade b/web/views/layout.jade index 732a22e..5e9f653 100644 --- a/web/views/layout.jade +++ b/web/views/layout.jade @@ -1,19 +1,19 @@ -!!! 5 +doctype html html head title Splunkbot Web Interface link(rel='stylesheet', href='/bootstrap/css/bootstrap.css') style(type='text/css') - body { - padding-top: 60px; - padding-bottom: 40px; - } + | body { + | padding-top: 60px; + | padding-bottom: 40px; + | } link(rel='stylesheet', href='/bootstrap/css/bootstrap-responsive.css') link(rel='stylesheet', href='/splunkbot/splunkbot.css') link(rel='stylesheet', href='/timeline.css') script(src='/bootstrap/js/jquery.js') script - serverTZOffset = #{serverTZOffset} + | serverTZOffset = #{serverTZOffset} body .navbar.navbar-fixed-top .navbar-inner @@ -60,6 +60,7 @@ html input(type='text', placeholder='#{placeholder}', name='q').search-query.span2 .container != body + block body hr footer p © Splunk 2012 @@ -78,4 +79,4 @@ html script(src='/splunkbot/splunkbot.js') script(src='/splunkbot/client/splunk.js') script(src='/spin.js') - script(src='/sprintf-0.7-beta1.js'); \ No newline at end of file + script(src='/sprintf-0.7-beta1.js') diff --git a/web/views/live.jade b/web/views/live.jade index be38dd9..bd08d59 100644 --- a/web/views/live.jade +++ b/web/views/live.jade @@ -1,29 +1,32 @@ -- var config_colors_script = 'var config_colors = '+JSON.stringify(colors) -script - !{config_colors_script}; - channel = "#{channel}"; +extends layout + +block body + - var config_colors_script = 'var config_colors = '+JSON.stringify(colors) + script + | !{config_colors_script}; + | channel = "#{channel}"; -.row - .span12 - .page-header - .row - h1.span9 Live results for #{channel} - div.btn-group.pull-right - - escapedchannel = encodeURIComponent(channel); - a(href='/live/#{escapedchannel}').btn.btn-success #{channel} - a(data-toggle='dropdown',href='#').btn.btn-success.dropdown-toggle - span.caret - ul.dropdown-menu - - for(var i=0;i#{search}" - - else - "" - | on #{channel} - div.pull-right - - if (typeof time === 'undefined') - div.btn-group - - var querystring = ""; - - if (typeof search !== 'undefined') - - querystring += 'q='+encodeURIComponent(search)+'&' - - var link = ''+count+'' - !{link} - - if (counts.length > 1) - a(data-toggle='dropdown',href='#').btn.btn-success.dropdown-toggle - span.caret - ul.dropdown-menu - - for(var i=0;i'+counts[i]+'' - !{link} - - else if (typeof time !== 'undefined') - div.btn-group - - var querystring = ""; - - if (typeof channel !== 'undefined') - - querystring += 'channel='+encodeURIComponent(channel)+'&' - - if (typeof time !== 'undefined') - - querystring += 'time='+time+'&' - - if (typeof highlight !== 'undefined') - - querystring += 'highlight='+encodeURIComponent(highlight)+'&' - - var timeidx = 0; - - for (var i=0; i'+times[timeidx][0]+'' - !{link} - - if (times.length > 1) - a(data-toggle='dropdown',href='#').btn.btn-success.dropdown-toggle - span.caret - ul.dropdown-menu - - for(var i=0;i'+times[i][0]+'' - !{link} - - if (typeof time === 'undefined') - p Click on a time to see the full log from that channel during that time - div(id='error').alert.alert-block.alert-error.fade.in.hide - a(data-dismiss='alert',href='#').close × - h4.alert-heading Fail! - p(id='errortext'). - Will be set by javascript - div(id='logbox').logbox - table - tbody(id='logboxtablebody') \ No newline at end of file + .row + .span12 + .page-header + .row + h1.span9 Search results for + - if (typeof time === 'undefined') + span(class="grey") #{search} + - else + span(class="grey",id="timestr") + | on #{channel} + div.pull-right + - if (typeof time === 'undefined') + div.btn-group + - var querystring = ""; + - if (typeof search !== 'undefined') + - querystring += 'q='+encodeURIComponent(search)+'&' + - var link = ''+count+'' + | !{link} + - if (counts.length > 1) + a(data-toggle='dropdown',href='#').btn.btn-success.dropdown-toggle + span.caret + ul.dropdown-menu + - for(var i=0;i'+counts[i]+'' + | !{link} + - else if (typeof time !== 'undefined') + div.btn-group + - var querystring = ""; + - if (typeof channel !== 'undefined') + - querystring += 'channel='+encodeURIComponent(channel)+'&' + - if (typeof time !== 'undefined') + - querystring += 'time='+time+'&' + - if (typeof highlight !== 'undefined') + - querystring += 'highlight='+encodeURIComponent(highlight)+'&' + - var timeidx = 0; + - for (var i=0; i'+times[timeidx][0]+'' + | !{link} + - if (times.length > 1) + a(data-toggle='dropdown',href='#').btn.btn-success.dropdown-toggle + span.caret + ul.dropdown-menu + - for(var i=0;i'+times[i][0]+'' + | !{link} + - if (typeof time === 'undefined') + p Click on a time to see the full log from that channel during that time + div(id='error').alert.alert-block.alert-error.fade.in.hide + a(data-dismiss='alert',href='#').close × + h4.alert-heading Fail! + p(id='errortext') + | Will be set by javascript + div(id='logbox').logbox + table + tbody(id='logboxtablebody') diff --git a/web/views/stats.jade b/web/views/stats.jade index 32c91c0..91ccd99 100644 --- a/web/views/stats.jade +++ b/web/views/stats.jade @@ -1,52 +1,55 @@ -script - var channel = "#{channel}"; +extends layout -.row - .span12 - .page-header - .row - h1.span5.offset4 - | Stats for #{channel} - div.btn-group.pull-right - - escapedchannel = encodeURIComponent(channel); - - var timeidx = 0; - - for (var i=0; i'+times[timeidx][0]+'' - !{link} - - if (times.length > 1) - a(data-toggle='dropdown',href='#').btn.btn-success.dropdown-toggle - span.caret - ul.dropdown-menu - - for(var i=0;i'+times[i][0]+'' - !{link} - div.btn-group.pull-left - - escapedchannel = encodeURIComponent(channel); - a(href='/stats/#{escapedchannel}').btn.btn-success #{channel} - a(data-toggle='dropdown',href='#').btn.btn-success.dropdown-toggle - span.caret - ul.dropdown-menu - - for(var i=0;i'+times[timeidx][0]+'' + | !{link} + - if (times.length > 1) + a(data-toggle='dropdown',href='#').btn.btn-success.dropdown-toggle + span.caret + ul.dropdown-menu + - for(var i=0;i'+times[i][0]+'' + | !{link} + div.btn-group.pull-left + - escapedchannel = encodeURIComponent(channel); + a(href='/stats/#{escapedchannel}').btn.btn-success #{channel} + a(data-toggle='dropdown',href='#').btn.btn-success.dropdown-toggle + span.caret + ul.dropdown-menu + - for(var i=0;i 1) - a(data-toggle='dropdown',href='#').btn.btn-success.dropdown-toggle - span.caret - ul(id='countmenu').dropdown-menu - - for(var i=0;i 1) - a(data-toggle='dropdown',href='#').btn.btn-success.dropdown-toggle - span.caret - ul(id='channelmenu').dropdown-menu - - for(var i=0;i 1) + a(data-toggle='dropdown',href='#').btn.btn-success.dropdown-toggle + span.caret + ul(id='countmenu').dropdown-menu + - for(var i=0;i 1) + a(data-toggle='dropdown',href='#').btn.btn-success.dropdown-toggle + span.caret + ul(id='channelmenu').dropdown-menu + - for(var i=0;i