Merge branch 'master' of github.com:ecdeveloper/node-web-crawler

Author: Evgheni Chiriliuc

app.js

@@ -2,15 +2,23 @@
  * Module dependencies.
  */
 
-var express   = require('express')
-  , http      = require('http')
-  , path      = require('path')
-  , partials  = require('express-partials')
-  , config    = require('./config')
-  , app       = express();
-
-global.io = require('socket.io').listen(app.listen( config.port ));
-
+var express    = require('express')
+  , http       = require('http')
+  , path       = require('path')
+  , partials   = require('express-partials')
+  , underscore = require('underscore')
+  , config     = require('./config')
+  , app        = express();
+
+// set config by environment
+if (process.env.ENVIRONMENT!='default') {
+    app.config = underscore.extend(config.default, config[process.env.ENVIRONMENT]);
+} else {
+    app.config = config.default;
+}
+
+// setup socket io
+global.io = require('socket.io').listen(app.listen( app.config.server.port ));
 io.configure(function () {
 	io.set('transports', ['websocket', 'xhr-polling']);
 	io.set('log level', config.log_level);
@@ -24,13 +32,13 @@ io.sockets.on('connection', function (socket)
 
 // db connect
 var db = require('mongoose');
-db.connect(config.db.service+'://'+config.db.host+'/'+config.db.database);
+db.connect(app.config.db.service+'://'+app.config.db.host+'/'+app.config.db.database);
 
 app.configure(function() {
 	app.set('views', __dirname + '/views');
 	app.set('view engine', 'ejs');
 	app.set('view options', { layout:true, pretty: true });
-	app.set('config', config);
+	app.set('config', app.config);
 	app.set('db', db);
 	app.use(express.favicon());
 	app.use(express.logger('dev'));

config.json

@@ -1,10 +1,19 @@
 /* Configurations */
 {
-  "log_level": 0,
-  "port": 3000,
-  "db": {
-    "service": "mongodb",
-    "host": "localhost",
-    "database": "search_for_404_v4"
+  "default": {
+    "log_level": 0,
+    "server": {
+      "scheme": "http",
+      "host": "localhost",
+      "port": 3000
+    },
+    "db": {
+      "service": "mongodb",
+      "host": "localhost",
+      "database": "search_for_404_v4"
+    }
+  },
+  "development": {
+    "log_level": 1
   }
 }

package.json

@@ -12,6 +12,8 @@
     "socket.io": "0.x.x",
     "named-argv": "0.x.x",
     "jsdom": "0.x.x",
-    "ejs": "0.x.x"
+    "ejs": "0.x.x",
+    "url": "0.x.x",
+    "underscore": "1.x.x"
   }
 }

routes/index.js

@@ -13,7 +13,7 @@ module.exports = function(app) {
 }
 
 function getHomePage(req, res) {
-	var port = res.app.settings.config.port;
+	var port = res.app.settings.config.server.port;
 
 	res.render('index', {
 		port: port
@@ -107,18 +107,28 @@ function postAddScraper(req, res)
 
 			case "sitemap-created":
 
-				var sitemap_path = "public/sitemaps/sitemap_"+ data.host +".xml";
-				fs.writeFile(sitemap_path, data.content, function(err)
-				{
-				    if(err)
-				        console.log(err);
-				   	else
-				        io.sockets.emit('sitemap-ready', {path: sitemap_path.replace("public/", "")})
+				var sitemap_path = "public/sitemaps/";
+				fs.exists(sitemap_path, function(exists) {
+					if (!exists) {
+						fs.mkdir(sitemap_path, writeSitemap);
+					} else {
+						writeSitemap();
+					}
 
-				    // Terminate crawling daemon
+					// Terminate crawling daemon
 					child.kill();
-				}); 
+				});
+				function writeSitemap() {
+					var filename = "sitemap_"+ data.host +".xml";
+					fs.writeFile(sitemap_path + filename, data.content, function(err)
+					{
+					    if(err)
+					        console.log(err);
+					   	else
+					        io.sockets.emit('sitemap-ready', {path: sitemap_path.replace("public/", "")})
+					});
 
+}
 				break;
 		}
 	})