Allow DNS challenges not just for cloudflare

Jenkinsfile

@@ -65,6 +65,7 @@ pipeline {
 				// See: https://github.com/yarnpkg/yarn/issues/3254
 				sh '''docker run --rm \\
 					-v "$(pwd)/backend:/app" \\
+					-v "$(pwd)/global:/app/global" \\
 					-w /app \\
 					node:latest \\
 					sh -c "yarn install && yarn eslint . && rm -rf node_modules"

backend/app.js

@@ -66,7 +66,7 @@ app.use(function (err, req, res, next) {
 		}
 	};
 
-	if (process.env.NODE_ENV === 'development') {
+	if (process.env.NODE_ENV === 'development' || (req.baseUrl + req.path).includes('nginx/certificates')) {
 		payload.debug = {
 			stack:    typeof err.stack !== 'undefined' && err.stack ? err.stack.split('\n') : null,
 			previous: err.previous

backend/config/sqlite-test-db.json

@@ -4,7 +4,7 @@
       "knex": {
         "client": "sqlite3",
         "connection": {
-          "filename": "/app/backend/config/mydb.sqlite"
+          "filename": "/app/config/mydb.sqlite"
         },
         "pool": {
           "min": 0,

backend/internal/certificate.js

@@ -13,6 +13,7 @@ const internalNginx    = require('./nginx');
 const internalHost     = require('./host');
 const certbot_command  = '/usr/bin/certbot';
 const le_config        = '/etc/letsencrypt.ini';
+const dns_plugins      = require('../global/certbot-dns-plugins');
 
 function omissions() {
 	return ['is_deleted'];
@@ -141,11 +142,11 @@ const internalCertificate = {
 								});
 						})
 						.then((in_use_result) => {
-							// Is CloudFlare, no config needed, so skip 3 and 5.
-							if (data.meta.cloudflare_use) {
+							// With DNS challenge no config is needed, so skip 3 and 5.
+							if (certificate.meta.dns_challenge) {
 								return internalNginx.reload().then(() => {
 									// 4. Request cert
-									return internalCertificate.requestLetsEncryptCloudFlareDnsSsl(certificate, data.meta.cloudflare_token);
+									return internalCertificate.requestLetsEncryptSslWithDnsChallenge(certificate);
 								})
 									.then(internalNginx.reload)
 									.then(() => {
@@ -772,35 +773,70 @@ const internalCertificate = {
 	},
 
 	/**
-	 * @param   {Object}  certificate   the certificate row
-	 * @param	{String} apiToken		the cloudflare api token
+	 * @param   {Object}  certificate   			the certificate row
+	 * @param		{String} 	dns_provider				the dns provider name (key used in `certbot-dns-plugins.js`)
+	 * @param		{String | null} 	credentials	the content of this providers credentials file
+	 * @param		{String} 	propagation_seconds	the cloudflare api token
 	 * @returns {Promise}
 	 */
-	requestLetsEncryptCloudFlareDnsSsl: (certificate, apiToken) => {
-		logger.info('Requesting Let\'sEncrypt certificates via Cloudflare DNS for Cert #' + certificate.id + ': ' + certificate.domain_names.join(', '));
+	requestLetsEncryptSslWithDnsChallenge: (certificate) => {
+		const dns_plugin = dns_plugins[certificate.meta.dns_provider];
+
+		if (!dns_plugin) {
+			throw Error(`Unknown DNS provider '${certificate.meta.dns_provider}'`);
+		}
+
+		logger.info(`Requesting Let'sEncrypt certificates via ${dns_plugin.display_name} for Cert #${certificate.id}: ${certificate.domain_names.join(', ')}`);
+
+		const credentials_loc = '/etc/letsencrypt/credentials-' + certificate.id;
+		const credentials_cmd = 'echo \'' + certificate.meta.dns_provider_credentials.replace('\'', '\\\'') + '\' > \'' + credentials_loc + '\' && chmod 600 \'' + credentials_loc + '\'';
+		const prepare_cmd     = 'pip3 install ' + dns_plugin.package_name + '==' + dns_plugin.package_version;
 
-		let tokenLoc = '~/cloudflare-token';
-		let storeKey = 'echo "dns_cloudflare_api_token = ' + apiToken + '" > ' + tokenLoc;	
+		// Whether the plugin has a --<name>-credentials argument
+		const has_config_arg = certificate.meta.dns_provider !== 'route53';
 
-		let cmd = 
-			storeKey + ' && ' +
+		let main_cmd = 
 			certbot_command + ' certonly --non-interactive ' +
 			'--cert-name "npm-' + certificate.id + '" ' +
 			'--agree-tos ' +
 			'--email "' + certificate.meta.letsencrypt_email + '" ' +			
 			'--domains "' + certificate.domain_names.join(',') + '" ' +
-			'--dns-cloudflare --dns-cloudflare-credentials ' + tokenLoc +
-			(le_staging ? ' --staging' : '')
-			+ ' && rm ' + tokenLoc;
+			'--authenticator ' + dns_plugin.full_plugin_name + ' ' +
+			(
+				has_config_arg 
+					? '--' + dns_plugin.full_plugin_name + '-credentials "' + credentials_loc + '"' 
+					: ''
+			) +
+			(
+				certificate.meta.propagation_seconds !== undefined 
+					? ' --' + dns_plugin.full_plugin_name + '-propagation-seconds ' + certificate.meta.propagation_seconds 
+					: ''
+			) +
+			(le_staging ? ' --staging' : '');
+
+		// Prepend the path to the credentials file as an environment variable
+		if (certificate.meta.dns_provider === 'route53') {
+			main_cmd = 'AWS_CONFIG_FILE=\'' + credentials_loc + '\' ' + main_cmd;
+		}
+
+		const teardown_cmd = `rm '${credentials_loc}'`;
 
 		if (debug_mode) {
-			logger.info('Command:', cmd);
+			logger.info('Command:', `${credentials_cmd} && ${prepare_cmd} && ${main_cmd} && ${teardown_cmd}`);
 		}
 
-		return utils.exec(cmd).then((result) => {
-			logger.info(result);
-			return result;
-		});
+		return utils.exec(credentials_cmd)
+			.then(() => {
+				return utils.exec(prepare_cmd)
+					.then(() => {
+						return utils.exec(main_cmd)
+							.then(async (result) => {
+								await utils.exec(teardown_cmd);
+								logger.info(result);
+								return result;
+							});
+					});
+			});
 	},
 
 
@@ -817,7 +853,7 @@ const internalCertificate = {
 			})
 			.then((certificate) => {
 				if (certificate.provider === 'letsencrypt') {
-					let renewMethod = certificate.meta.cloudflare_use ? internalCertificate.renewLetsEncryptCloudFlareSsl : internalCertificate.renewLetsEncryptSsl;		
+					let renewMethod = certificate.meta.dns_challenge ? internalCertificate.renewLetsEncryptSslWithDnsChallenge : internalCertificate.renewLetsEncryptSsl;		
 
 					return renewMethod(certificate)
 						.then(() => {
@@ -877,22 +913,47 @@ const internalCertificate = {
 	 * @param   {Object}  certificate   the certificate row
 	 * @returns {Promise}
 	 */
-	renewLetsEncryptCloudFlareSsl: (certificate) => {
-		logger.info('Renewing Let\'sEncrypt certificates for Cert #' + certificate.id + ': ' + certificate.domain_names.join(', '));
+	renewLetsEncryptSslWithDnsChallenge: (certificate) => {
+		const dns_plugin = dns_plugins[certificate.meta.dns_provider];
 
-		let cmd = certbot_command + ' renew --non-interactive ' +
+		if (!dns_plugin) {
+			throw Error(`Unknown DNS provider '${certificate.meta.dns_provider}'`);
+		}
+
+		logger.info(`Renewing Let'sEncrypt certificates via ${dns_plugin.display_name} for Cert #${certificate.id}: ${certificate.domain_names.join(', ')}`);
+
+		const credentials_loc = '/etc/letsencrypt/credentials-' + certificate.id;
+		const credentials_cmd = 'echo \'' + certificate.meta.dns_provider_credentials.replace('\'', '\\\'') + '\' > \'' + credentials_loc + '\' && chmod 600 \'' + credentials_loc + '\'';
+		const prepare_cmd     = 'pip3 install ' + dns_plugin.package_name + '==' + dns_plugin.package_version;
+
+		let main_cmd = 
+			certbot_command + ' renew --non-interactive ' +
 			'--cert-name "npm-' + certificate.id + '" ' +
-			'--disable-hook-validation ' +
-			(le_staging ? '--staging' : '');
+			'--disable-hook-validation' +
+			(le_staging ? ' --staging' : '');
+
+		// Prepend the path to the credentials file as an environment variable
+		if (certificate.meta.dns_provider === 'route53') {
+			main_cmd = 'AWS_CONFIG_FILE=\'' + credentials_loc + '\' ' + main_cmd;
+		}
+
+		const teardown_cmd = `rm '${credentials_loc}'`;
 
 		if (debug_mode) {
-			logger.info('Command:', cmd);
+			logger.info('Command:', `${credentials_cmd} && ${prepare_cmd} && ${main_cmd} && ${teardown_cmd}`);
 		}
 
-		return utils.exec(cmd)
-			.then((result) => {
-				logger.info(result);
-				return result;
+		return utils.exec(credentials_cmd)
+			.then(() => {
+				return utils.exec(prepare_cmd)
+					.then(() => {
+						return utils.exec(main_cmd)
+							.then(async (result) => {
+								await utils.exec(teardown_cmd);
+								logger.info(result);
+								return result;
+							});
+					});
 			});
 	},
 

backend/routes/api/nginx/certificates.js

@@ -58,6 +58,7 @@ router
 	.post((req, res, next) => {
 		apiValidator({$ref: 'endpoints/certificates#/links/1/schema'}, req.body)
 			.then((payload) => {
+				req.setTimeout(900000); // 15 minutes timeout
 				return internalCertificate.create(res.locals.access, payload);
 			})
 			.then((result) => {
@@ -197,6 +198,7 @@ router
 	 * Renew certificate
 	 */
 	.post((req, res, next) => {
+		req.setTimeout(900000); // 15 minutes timeout
 		internalCertificate.renew(res.locals.access, {
 			id: parseInt(req.params.certificate_id, 10)
 		})

backend/schema/endpoints/certificates.json

@@ -42,11 +42,23 @@
         "letsencrypt_agree": {
           "type": "boolean"
         },
-        "cloudflare_use": {
+        "dns_challenge": {
           "type": "boolean"
         },
-        "cloudflare_token": {
+        "dns_provider": {
           "type": "string"
+        },
+        "dns_provider_credentials": {
+          "type": "string"
+        },
+        "propagation_seconds": {
+          "anyOf": [
+            { 
+              "type": "integer",
+              "minimum": 0 
+            }
+          ]
+
         }
       }
     }

docker/Dockerfile

@@ -17,8 +17,8 @@ ENV NODE_ENV=production
 
 RUN echo "fs.file-max = 65535" > /etc/sysctl.conf \
 	&& apk update \
-	&& apk add python2 py-pip certbot jq \
-	&& pip install certbot-dns-cloudflare \
+	&& apk add python3 certbot jq \
+	&& python3 -m ensurepip \
 	&& rm -rf /var/cache/apk/*
 
 ENV NPM_BUILD_VERSION="${BUILD_VERSION}" NPM_BUILD_COMMIT="${BUILD_COMMIT}" NPM_BUILD_DATE="${BUILD_DATE}"
@@ -34,6 +34,7 @@ EXPOSE 443
 COPY docker/rootfs      /
 ADD backend             /app
 ADD frontend/dist       /app/frontend
+COPY global							/app/global
 
 WORKDIR /app
 RUN yarn install

docker/dev/Dockerfile

@@ -7,8 +7,8 @@ ENV S6_FIX_ATTRS_HIDDEN=1
 
 RUN echo "fs.file-max = 65535" > /etc/sysctl.conf \
 	&& apk update \
-	&& apk add python2 py-pip certbot jq \
-	&& pip install certbot-dns-cloudflare \
+	&& apk add python3 certbot jq \
+	&& python3 -m ensurepip \
 	&& rm -rf /var/cache/apk/*
 
 # Task

docker/docker-compose.dev.yml

@@ -11,6 +11,8 @@ services:
       - 3080:80
       - 3081:81
       - 3443:443
+    networks:
+      - nginx_proxy_manager
     environment:
       - NODE_ENV=development
       - FORCE_COLOR=1
@@ -19,13 +21,17 @@ services:
     volumes:
       - npm_data:/data
       - le_data:/etc/letsencrypt
-      - ..:/app
+      - ../backend:/app
+      - ../frontend:/app/frontend
+      - ../global:/app/global
     depends_on:
       - db
     working_dir: /app
 
   db:
     image: jc21/mariadb-aria
+    networks:
+      - nginx_proxy_manager
     environment:
       MYSQL_ROOT_PASSWORD: "npm"
       MYSQL_DATABASE: "npm"
@@ -38,6 +44,8 @@ services:
     image: 'swaggerapi/swagger-ui:latest'
     ports:
       - 3001:80
+    networks:
+      - nginx_proxy_manager
     environment:
       URL: "http://127.0.0.1:3081/api/schema"
       PORT: '80'
@@ -48,3 +56,6 @@ volumes:
   npm_data:
   le_data:
   db_data:
+
+networks:
+  nginx_proxy_manager:

docker/rootfs/etc/nginx/conf.d/dev.conf

@@ -17,6 +17,9 @@ server {
 		proxy_set_header      X-Forwarded-Proto  $scheme;
 		proxy_set_header      X-Forwarded-For    $remote_addr;
 		proxy_pass            http://127.0.0.1:3000/;
+
+		proxy_read_timeout 15m;
+		proxy_send_timeout 15m;
 	}
 
 	___location / {

docker/rootfs/etc/nginx/conf.d/production.conf

@@ -18,6 +18,9 @@ server {
 		proxy_set_header      X-Forwarded-Proto  $scheme;
 		proxy_set_header      X-Forwarded-For    $remote_addr;
 		proxy_pass            http://127.0.0.1:3000/;
+
+		proxy_read_timeout 15m;
+		proxy_send_timeout 15m;
 	}
 
 	___location / {

docker/rootfs/etc/services.d/manager/run

@@ -5,7 +5,7 @@ mkdir -p /data/letsencrypt-acme-challenge
 cd /app || echo
 
 if [ "$DEVELOPMENT" == "true" ]; then
-	cd /app/backend || exit 1
+	cd /app || exit 1
 	yarn install
 	node --max_old_space_size=250 --abort_on_uncaught_exception node_modules/nodemon/bin/nodemon.js
 else

frontend/js/app/api.js

@@ -53,7 +53,7 @@ function fetch(verb, path, data, options) {
             contentType: options.contentType || 'application/json; charset=UTF-8',
             processData: options.processData || true,
             crossDomain: true,
-            timeout:     options.timeout ? options.timeout : 30000,
+            timeout:     options.timeout ? options.timeout : 180000,
             xhrFields:   {
                 withCredentials: true
             },
@@ -587,7 +587,8 @@ module.exports = {
              * @param {Object}  data
              */
             create: function (data) {
-                return fetch('post', 'nginx/certificates', data);
+                const timeout = 180000 + (data.meta.propagation_seconds ? Number(data.meta.propagation_seconds) * 1000 : 0);
+                return fetch('post', 'nginx/certificates', data, {timeout});
             },
 
             /**
@@ -630,8 +631,8 @@ module.exports = {
              * @param   {Number}  id
              * @returns {Promise}
              */
-            renew: function (id) {
-                return fetch('post', 'nginx/certificates/' + id + '/renew');
+            renew: function (id, timeout = 180000) {
+                return fetch('post', 'nginx/certificates/' + id + '/renew', undefined, {timeout});
             }
         }
     },