Merge pull request #7 from NetCrafters/feature-config-file

Evgheni C. · Evgheni C. · commit 4c19023a2410 · 2013-07-23T13:54:08.000-07:00
Extract database config to a file and share db with child.
diff --git a/app.js b/app.js
@@ -6,13 +6,14 @@ var express   = require('express')
   , http      = require('http')
   , path      = require('path')
   , partials  = require('express-partials')
-  , app		  = express();
+  , config    = require('./config')
+  , app       = express();
 
-global.io = require('socket.io').listen(app.listen( 3000 ));
+global.io = require('socket.io').listen(app.listen( config.port ));
 
 io.configure(function () {
 	io.set('transports', ['websocket', 'xhr-polling']);
-	io.set('log level', 0);
+	io.set('log level', config.log_level);
 	io.set('force new connection', true);
 });
 
@@ -21,10 +22,16 @@ io.sockets.on('connection', function (socket)
 	socket.on('setMaxThreads', function(data){  });
 });
 
+// db connect
+var db = require('mongoose');
+db.connect(config.db.service+'://'+config.db.host+'/'+config.db.database);
+
 app.configure(function() {
 	app.set('views', __dirname + '/views');
 	app.set('view engine', 'ejs');
 	app.set('view options', { layout:true, pretty: true });
+	app.set('config', config);
+	app.set('db', db);
 	app.use(express.favicon());
 	app.use(express.logger('dev'));
 	app.use(express.bodyParser());
@@ -39,3 +46,4 @@ app.configure('development', function(){
 });
 
 require('./routes')(app);
+
diff --git a/config.json b/config.json
@@ -0,0 +1,9 @@
+{
+  "log_level": 0,
+  "port": 3000,
+  "db": {
+    "service": "mongodb",
+    "host": "localhost",
+    "database": "search_for_404_v4"
+  }
+}
diff --git a/crawling-daemon.js b/crawling-daemon.js
@@ -2,28 +2,41 @@
 	Scraper app. Only runs as child forked process.
 */
 
-var	util 	= require('util'),
-	http 	= require('http'),
-	url 	= require('url'),
-	mongoose= require('mongoose'),
-	argv 	= require('named-argv'),
-	jsdom	= require('jsdom');
-
-mongoose.connect('mongodb://localhost/search_for_404_v4');
+var	util 	 = require('util'),
+	http 	 = require('http'),
+	url 	 = require('url'),
+	mongoose = require('mongoose'),
+	argv 	 = require('named-argv'),
+	jsdom	 = require('jsdom');
 
 /*
 	requestsRunningPool: array of links are requesting now
 */
 
-var scrapeHost = "", max_depth, create_sitemap, link, auth = {},
-	LinksCheckModel, LinksGrabbedModel,
-	requestsRunning = 0, requestsRunningPool = [], requestsPerSecond = 0, maxThreads = 5,
-	checkUrlInterval = null, processingDOM = false;
+var scrapeHost = "",
+	max_depth,
+	create_sitemap,
+	link,
+	auth = {},
+	LinksCheckModel,
+	LinksGrabbedModel,
+	requestsRunning = 0,
+	requestsRunningPool = [],
+	requestsPerSecond = 0,
+	maxThreads = 5,
+	checkUrlInterval = null,
+	processingDOM = false,
+	config;
 
 process.on("message", function(data)
 {
 	switch (data.action)
 	{
+		case "setConfig":
+			config = data.config;
+			mongoose.connect(config.db.service+'://'+config.db.host+'/'+config.db.database);
+			break;
+
 		case "setAuth":
 			auth.user = data.auth_user;
 			auth.pass = data.auth_pass;
diff --git a/routes/index.js b/routes/index.js
@@ -13,7 +13,11 @@ module.exports = function(app) {
 }
 
 function getHomePage(req, res) {
-	res.render('index');
+	var port = res.app.settings.config.port;
+
+	res.render('index', {
+		port: port
+	});
 }
 
 function postAddScraper(req, res)
@@ -23,9 +27,16 @@ function postAddScraper(req, res)
 		auth_pass	   = req.body.auth_pass,
 		depth 		   = parseInt(req.body.create_crawler_depth),
 		create_sitemap = req.body.create_crawler_sitemap == 1,
-		clean 		   = req.body.clean_crawl == 1;
+		clean 		   = req.body.clean_crawl == 1,
+		config         = res.app.settings.config;
 
 	var child = child_process.fork("crawling-daemon.js");
+
+	// setup config
+	child.send({
+		action: "setConfig",
+		config: config
+	});
 	
 	if (auth_user!="" && auth_pass!="")
 		child.send(
@@ -114,4 +125,4 @@ function postAddScraper(req, res)
 
 	// child_processes[url] = child;
 	res.redirect("/");
-}
+}
diff --git a/views/layout.ejs b/views/layout.ejs
@@ -2,7 +2,7 @@
 <html lang="en">
   <head>
 	<meta charset="utf-8">
-	<title>Bootstrap, from Twitter</title>
+	<title>Node Web Crawler</title>
 	<meta name="viewport" content="width=device-width, initial-scale=1.0">
 	<meta name="description" content="">
 	<meta name="author" content="">
@@ -17,7 +17,7 @@
 	<script src="/socket.io/socket.io.js"></script>
 
 	<script>
-		var socket = io.connect(window.___location.hostname, {port: 3000});
+		var socket = io.connect(window.___location.hostname, {port: <%= port %>});
 	</script>
 </head>
 
