diff --git a/app.js b/app.js index 58e5b23..cdfd61e 100644 --- a/app.js +++ b/app.js @@ -6,13 +6,14 @@ var express = require('express') , http = require('http') , path = require('path') , partials = require('express-partials') - , app = express(); + , config = require('./config') + , app = express(); -global.io = require('socket.io').listen(app.listen( 3000 )); +global.io = require('socket.io').listen(app.listen( config.port )); io.configure(function () { io.set('transports', ['websocket', 'xhr-polling']); - io.set('log level', 0); + io.set('log level', config.log_level); io.set('force new connection', true); }); @@ -21,10 +22,16 @@ io.sockets.on('connection', function (socket) socket.on('setMaxThreads', function(data){ }); }); +// db connect +var db = require('mongoose'); +db.connect(config.db.service+'://'+config.db.host+'/'+config.db.database); + app.configure(function() { app.set('views', __dirname + '/views'); app.set('view engine', 'ejs'); app.set('view options', { layout:true, pretty: true }); + app.set('config', config); + app.set('db', db); app.use(express.favicon()); app.use(express.logger('dev')); app.use(express.bodyParser()); @@ -39,3 +46,4 @@ app.configure('development', function(){ }); require('./routes')(app); + diff --git a/config.json b/config.json new file mode 100644 index 0000000..acd8224 --- /dev/null +++ b/config.json @@ -0,0 +1,9 @@ +{ + "log_level": 0, + "port": 3000, + "db": { + "service": "mongodb", + "host": "localhost", + "database": "search_for_404_v4" + } +} diff --git a/crawling-daemon.js b/crawling-daemon.js index 70d62c1..0f94969 100644 --- a/crawling-daemon.js +++ b/crawling-daemon.js @@ -2,28 +2,41 @@ Scraper app. Only runs as child forked process. */ -var util = require('util'), - http = require('http'), - url = require('url'), - mongoose= require('mongoose'), - argv = require('named-argv'), - jsdom = require('jsdom'); - -mongoose.connect('mongodb://localhost/search_for_404_v4'); +var util = require('util'), + http = require('http'), + url = require('url'), + mongoose = require('mongoose'), + argv = require('named-argv'), + jsdom = require('jsdom'); /* requestsRunningPool: array of links are requesting now */ -var scrapeHost = "", max_depth, create_sitemap, link, auth = {}, - LinksCheckModel, LinksGrabbedModel, - requestsRunning = 0, requestsRunningPool = [], requestsPerSecond = 0, maxThreads = 5, - checkUrlInterval = null, processingDOM = false; +var scrapeHost = "", + max_depth, + create_sitemap, + link, + auth = {}, + LinksCheckModel, + LinksGrabbedModel, + requestsRunning = 0, + requestsRunningPool = [], + requestsPerSecond = 0, + maxThreads = 5, + checkUrlInterval = null, + processingDOM = false, + config; process.on("message", function(data) { switch (data.action) { + case "setConfig": + config = data.config; + mongoose.connect(config.db.service+'://'+config.db.host+'/'+config.db.database); + break; + case "setAuth": auth.user = data.auth_user; auth.pass = data.auth_pass; diff --git a/routes/index.js b/routes/index.js index d41d1d5..8209f98 100644 --- a/routes/index.js +++ b/routes/index.js @@ -13,7 +13,11 @@ module.exports = function(app) { } function getHomePage(req, res) { - res.render('index'); + var port = res.app.settings.config.port; + + res.render('index', { + port: port + }); } function postAddScraper(req, res) @@ -23,9 +27,16 @@ function postAddScraper(req, res) auth_pass = req.body.auth_pass, depth = parseInt(req.body.create_crawler_depth), create_sitemap = req.body.create_crawler_sitemap == 1, - clean = req.body.clean_crawl == 1; + clean = req.body.clean_crawl == 1, + config = res.app.settings.config; var child = child_process.fork("crawling-daemon.js"); + + // setup config + child.send({ + action: "setConfig", + config: config + }); if (auth_user!="" && auth_pass!="") child.send( @@ -114,4 +125,4 @@ function postAddScraper(req, res) // child_processes[url] = child; res.redirect("/"); -} \ No newline at end of file +} diff --git a/views/layout.ejs b/views/layout.ejs index 570a69c..8eeea69 100644 --- a/views/layout.ejs +++ b/views/layout.ejs @@ -2,7 +2,7 @@
-