Buildx improvements

Author: jc21

.dockerignore

@@ -0,0 +1,7 @@
+# Ignore everything
+*
+
+# Only allow the following for docker build:
+!backend/
+!docker/
+!scripts/

Jenkinsfile

@@ -1,6 +1,6 @@
 pipeline {
 	agent {
-		label 'docker-multiarch'
+		label 'taurus'
 	}
 	options {
 		buildDiscarder(logRotator(numToKeepStr: '5'))
@@ -71,18 +71,16 @@ pipeline {
 			steps {
 				withCredentials([usernamePassword(credentialsId: 'oss-index-token', passwordVariable: 'NANCY_TOKEN', usernameVariable: 'NANCY_USER')]) {
 					sh '''docker build --pull --no-cache --squash --compress \\
-						-t ${IMAGE}:ci-${BUILD_NUMBER} \\
+						-t "${IMAGE}:${BRANCH_LOWER}-ci-${BUILD_NUMBER}" \\
 						-f docker/Dockerfile \\
-						--build-arg TARGETPLATFORM=linux/amd64 \\
-						--build-arg BUILDPLATFORM=linux/amd64 \\
+						--build-arg BUILD_COMMIT="${BUILD_COMMIT:-dev}" \\
 						--build-arg BUILD_DATE="$(date '+%Y-%m-%d %T %Z')" \\
 						--build-arg BUILD_VERSION="${BUILD_VERSION}" \\
-						--build-arg BUILD_COMMIT="${BUILD_COMMIT}" \\
-						--build-arg SENTRY_DSN="${SENTRY_DSN:-}" \\
-						--build-arg GOPROXY="${GOPROXY:-}" \\
 						--build-arg GOPRIVATE="${GOPRIVATE:-}" \\
-						--build-arg NANCY_USER="${NANCY_USER}" \\
-						--build-arg NANCY_TOKEN="${NANCY_TOKEN}" \\
+						--build-arg GOPROXY="${GOPROXY:-}" \\
+						--build-arg NANCY_TOKEN="${NANCY_TOKEN:-}" \\
+						--build-arg NANCY_USER="${NANCY_USER:-}" \\
+						--build-arg SENTRY_DSN="${SENTRY_DSN:-}" \\
 						.
 					'''
 				}
@@ -151,9 +149,9 @@ pipeline {
 				withCredentials([string(credentialsId: 'npm-sentry-dsn', variable: 'SENTRY_DSN')]) {
 					withCredentials([usernamePassword(credentialsId: 'jc21-dockerhub', passwordVariable: 'dpass', usernameVariable: 'duser')]) {
 						// Docker Login
-						sh "docker login -u '${duser}' -p '${dpass}'"
+						sh 'docker login -u "${duser}" -p "${dpass}"'
 						// Buildx with push from cache
-						sh "./scripts/buildx --push ${BUILDX_PUSH_TAGS}"
+						sh './scripts/buildx --push ${BUILDX_PUSH_TAGS}'
 						// sh './scripts/buildx -o type=local,dest=docker-build'
 					}
 				}
@@ -206,34 +204,6 @@ pipeline {
 				}
 			}
 		}
-		/*
-		stage('Artifacts') {
-			when {
-				allOf {
-					not {
-						equals expected: 'UNSTABLE', actual: currentBuild.result
-					}
-				}
-			}
-			steps {
-				sh 'mkdir -p artifacts'
-				// Multiarch builds
-				dir(path: 'docker-build/linux_amd64/app') {
-					sh 'zip -qr ../../../artifacts/linux_amd64.zip *'
-				}
-				dir(path: 'docker-build/linux_arm64/app') {
-					sh 'zip -qr ../../../artifacts/linux_arm64.zip *'
-				}
-				dir(path: 'docker-build/linux_arm_v7/app') {
-					sh 'zip -qr ../../../artifacts/linux_arm_v7.zip *'
-				}
-				// Archive them
-				dir(path: 'artifacts') {
-					archiveArtifacts artifacts: '** /*'
-				}
-			}
-		}
-		*/
 	}
 	post {
 		always {

backend/go.sum

@@ -69,18 +69,6 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/
 github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk=
 github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g=
 github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
-github.com/jc21/jsref v0.0.0-20210608013137-43b07c7d31bd h1:Ag/L5Yc9BeBbi4i8bNAev8Ejtu/jq8Qk/xK+HDHnWNc=
-github.com/jc21/jsref v0.0.0-20210608013137-43b07c7d31bd/go.mod h1:yIq2t51OJgVsdRlPY68NAnyVdBH0kYXxDTFtUxOap80=
-github.com/jc21/jsref v0.0.0-20210608014024-8bda7cb41eef h1:1jF5nv8PmgH2txfWGmsPium0Hj9PEnGkb96tkZ+4uDU=
-github.com/jc21/jsref v0.0.0-20210608014024-8bda7cb41eef/go.mod h1:yIq2t51OJgVsdRlPY68NAnyVdBH0kYXxDTFtUxOap80=
-github.com/jc21/jsref v0.0.0-20210608014914-2edd4dea9791 h1:s0hsMFnTiGGytgwDbHo20OvmJj2/+FFMZvLpRNexnvk=
-github.com/jc21/jsref v0.0.0-20210608014914-2edd4dea9791/go.mod h1:yIq2t51OJgVsdRlPY68NAnyVdBH0kYXxDTFtUxOap80=
-github.com/jc21/jsref v0.0.0-20210608023003-123d7fb98643 h1:ZpDTP4ow7hZMx0ORi06jnLP4ZDGQVa6SayH+5rWWlYg=
-github.com/jc21/jsref v0.0.0-20210608023003-123d7fb98643/go.mod h1:yIq2t51OJgVsdRlPY68NAnyVdBH0kYXxDTFtUxOap80=
-github.com/jc21/jsref v0.0.0-20210608023437-810a57e5f736 h1:1nZYRLsHvECy8rbOLkqRBK45Y6zKQ5ZRuGPMQalPWVc=
-github.com/jc21/jsref v0.0.0-20210608023437-810a57e5f736/go.mod h1:yIq2t51OJgVsdRlPY68NAnyVdBH0kYXxDTFtUxOap80=
-github.com/jc21/jsref v0.0.0-20210608024103-9eaa65f76123 h1:pb24Ybg78OdqO4GHh0xcwlVPWKlDYX/ZVnf+wq8D9To=
-github.com/jc21/jsref v0.0.0-20210608024103-9eaa65f76123/go.mod h1:yIq2t51OJgVsdRlPY68NAnyVdBH0kYXxDTFtUxOap80=
 github.com/jc21/jsref v0.0.0-20210608024405-a97debfc4760 h1:7wxq2DIgtO36KLrFz1RldysO0WVvcYsD49G9tyAs01k=
 github.com/jc21/jsref v0.0.0-20210608024405-a97debfc4760/go.mod h1:yIq2t51OJgVsdRlPY68NAnyVdBH0kYXxDTFtUxOap80=
 github.com/jmoiron/sqlx v1.3.3 h1:j82X0bf7oQ27XeqxicSZsTU5suPwKElg3oyxNn43iTk=

docker/Dockerfile

@@ -11,59 +11,40 @@ FROM jc21/nginx-full:github-acme.sh-golang AS gobuild
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-ARG GOPROXY
+ARG BUILD_COMMIT
+ARG BUILD_VERSION
 ARG GOPRIVATE
-
-ENV GOPROXY=$GOPROXY \
-	GOPRIVATE=$GOPRIVATE \
-	GO111MODULE=on \
-	CGO_ENABLED=1
-
-# Nancy
-RUN go get github.com/sonatype-nexus-community/nancy
-RUN mkdir -p /workspace
-WORKDIR /workspace
-COPY backend/go.mod backend/go.sum backend/.nancy-ignore ./
-RUN go mod download
-
+ARG GOPROXY
 ARG NANCY_TOKEN
 ARG NANCY_USER
-RUN go list -json -m all | nancy sleuth --quiet --username "${NANCY_USER}" --token "${NANCY_TOKEN}"
-RUN rm -rf /workspace
+ARG SENTRY_DSN
+ARG SKIP_TESTS
+
+ENV BUILD_COMMIT="${BUILD_COMMIT:-dev}" \
+	BUILD_VERSION="${BUILD_VERSION:-0.0.0}" \
+	CGO_ENABLED=1 \
+	GO111MODULE=on \
+	GOPRIVATE="${GOPRIVATE:-}" \
+	GOPROXY="${GOPROXY:-}" \
+	NANCY_TOKEN="${NANCY_TOKEN:-}" \
+	NANCY_USER="${NANCY_USER:-}" \
+	SENTRY_DSN="${SENTRY_DSN:-}" \
+	SKIP_TESTS="${SKIP_TESTS:-}"
 
 # Code
+RUN mkdir -p /app
 WORKDIR /app
 COPY . .
-WORKDIR /app/backend
-
-# Build
-RUN go mod download
-RUN echo "Testing and compiling project" \
-	&& [ -z "$(go tool fix -diff ./internal)" ]
-
-# Disabled as CI has issues at the moment
-#RUN if [ "$TARGETPLATFORM" == "" ] || [ "$TARGETPLATFORM" == "linux/amd64" ]; then golangci-lint -v run ./...; fi
-
-RUN richgo test -cover -v ./internal/...
-RUN richgo test -bench=. ./internal/...
-
-ARG BUILD_VERSION
-ARG BUILD_COMMIT
-ARG SENTRY_DSN
-RUN go build \
-	-ldflags "-w -s -X main.commit=${BUILD_COMMIT} -X main.version=${BUILD_VERSION} -X main.sentryDSN=${SENTRY_DSN:-}" \
-	-o ../dist/bin/server \
-	-v ./cmd/server
+RUN ./scripts/docker-gobuild
 
 #===============
 # Final image
 #===============
 
-FROM jc21/nginx-full:github-acme.sh
+FROM jc21/nginx-full:github-acme.sh AS final
 
 COPY --from=gobuild /app/dist /app
 COPY --from=gobuild /app/backend/migrations /app/migrations
-# COPY frontend/build /app/frontend
 
 ENV SUPPRESS_NO_CONFIG_WARNING=1
 ENV S6_FIX_ATTRS_HIDDEN=1
@@ -83,25 +64,29 @@ RUN rm -rf /etc/services.d/frontend /etc/nginx/conf.d/dev.conf
 VOLUME /data
 
 CMD [ "/init" ]
+# TODO: remove healthchecks
 HEALTHCHECK --interval=15s --timeout=3s CMD curl -f http://127.0.0.1:81/api || exit 1
 
 ARG NOW
 ARG BUILD_VERSION
 ARG BUILD_COMMIT
 ARG BUILD_DATE
-ENV NPM_BUILD_VERSION="${BUILD_VERSION}" NPM_BUILD_COMMIT="${BUILD_COMMIT}" NPM_BUILD_DATE="${BUILD_DATE}"
+
 ENV DATABASE_URL="sqlite:////data/nginxproxymanager.db" \
 	DBMATE_MIGRATIONS_DIR="/app/migrations" \
+	DBMATE_NO_DUMP_SCHEMA="1" \
 	DBMATE_SCHEMA_FILE="/data/schema.sql" \
-	DBMATE_NO_DUMP_SCHEMA="1"
+	NPM_BUILD_VERSION="${BUILD_VERSION:-0.0.0}" \
+	NPM_BUILD_COMMIT="${BUILD_COMMIT:-dev}" \
+	NPM_BUILD_DATE="${BUILD_DATE:-}"
 
 LABEL org.label-schema.schema-version="1.0" \
 	org.label-schema.license="MIT" \
 	org.label-schema.name="nginx-proxy-manager" \
 	org.label-schema.description="Nginx Host Management and Proxy" \
-	org.label-schema.build-date="$NOW" \
-	org.label-schema.version="$BUILD_VERSION" \
+	org.label-schema.build-date="${NOW:-}" \
+	org.label-schema.version="${BUILD_VERSION:-0.0.0}" \
 	org.label-schema.url="https://nginxproxymanager.com" \
 	org.label-schema.vcs-url="https://github.com/jc21/nginx-proxy-manager.git" \
-	org.label-schema.vcs-ref="$BUILD_COMMIT" \
-	org.label-schema.cmd="docker run --rm -ti jc21/nginx-proxy-manager:$BUILD_VERSION"
+	org.label-schema.vcs-ref="${BUILD_COMMIT:-dev}" \
+	org.label-schema.cmd="docker run --rm -ti jc21/nginx-proxy-manager:${BUILD_VERSION:-0.0.0}"

docker/docker-compose.ci.yml

@@ -3,7 +3,7 @@ version: "3"
 services:
 
   fullstack:
-    image: ${IMAGE}:ci-${BUILD_NUMBER}
+    image: ${IMAGE}:${BRANCH_LOWER}-ci-${BUILD_NUMBER}
     environment:
       - LOG_LEVEL=debug
     volumes:

scripts/buildx

@@ -17,12 +17,13 @@ docker buildx create --name "${BUILDX_NAME:-npm}" || echo
 docker buildx use "${BUILDX_NAME:-npm}"
 
 docker buildx build \
-	--build-arg BUILD_VERSION="${BUILD_VERSION:-dev}" \
 	--build-arg BUILD_COMMIT="${BUILD_COMMIT:-notset}" \
 	--build-arg BUILD_DATE="$(date '+%Y-%m-%d %T %Z')" \
+	--build-arg BUILD_VERSION="${BUILD_VERSION:-dev}" \
 	--build-arg NOW="$(date --rfc-3339=s)" \
-	--build-arg GOPROXY="${GOPROXY:-}" \
+	--build-arg SKIP_TESTS=1 \
 	--build-arg GOPRIVATE="${GOPRIVATE:-}" \
+	--build-arg GOPROXY="${GOPROXY:-}" \
 	--build-arg SENTRY_DSN="${SENTRY_DSN:-}" \
 	--platform linux/amd64,linux/arm64,linux/arm/7 \
 	--progress plain \

scripts/docker-gobuild

@@ -0,0 +1,50 @@
+#!/bin/bash -e
+
+# This script is run as part of the Dockerfile
+# It will conduct golang testing and vuln lookups
+# unless SKIP_TESTS=1 is defined
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. "$DIR/.common.sh"
+
+echo -e "${BLUE}❯ ${CYAN}docker-gobuild${RESET}"
+echo -e "  ${YELLOW}BUILD_COMMIT:  ${BUILD_COMMIT:-not set}${RESET}"
+echo -e "  ${YELLOW}BUILD_VERSION: ${BUILD_VERSION:-not set}${RESET}"
+echo -e "  ${YELLOW}CGO_ENABLED:   ${CGO_ENABLED:-not set}${RESET}"
+echo -e "  ${YELLOW}GOPROXY:       ${GOPROXY:-not set}${RESET}"
+echo -e "  ${YELLOW}GOPRIVATE:     ${GOPRIVATE:-not set}${RESET}"
+echo -e "  ${YELLOW}GO111MODULE:   ${GO111MODULE:-not set}${RESET}"
+echo -e "  ${YELLOW}SKIP_TESTS:    ${SKIP_TESTS:-not set}${RESET}"
+
+echo -e "${BLUE}❯ ${CYAN}Downloading backend go modules${RESET}"
+cd /app/backend
+go mod download
+
+# Testing and vulnerability lookup
+if ! [ "${SKIP_TESTS:-}" = "1" ];  then
+	mkdir -p /workspace
+	echo -e "${BLUE}❯ ${CYAN}Nancy setup${RESET}"
+	cd /workspace
+	go get github.com/sonatype-nexus-community/nancy
+	cp /app/backend/go.mod /app/backend/go.sum /app/backend/.nancy-ignore .
+	go mod download
+
+	echo -e "${BLUE}❯ ${CYAN}Nancy testing${RESET}"
+	go list -json -m all | nancy sleuth --quiet --username "${NANCY_USER}" --token "${NANCY_TOKEN:-}"
+	rm -rf /workspace
+
+	echo -e "${BLUE}❯ ${CYAN}Testing backend code${RESET}"
+	cd /app/backend
+	[ -z "$(go tool fix -diff ./internal)" ]
+	richgo test -cover -v ./internal/...
+	richgo test -bench=. ./internal/...
+	golangci-lint -v run ./...
+fi
+
+echo -e "${BLUE}❯ ${CYAN}Building backend binary${RESET}"
+go build \
+	-ldflags "-w -s -X main.commit=${BUILD_COMMIT} -X main.version=${BUILD_VERSION} -X main.sentryDSN=${SENTRY_DSN:-}" \
+	-o ../dist/bin/server \
+	-v ./cmd/server
+
+echo -e "${BLUE}❯ ${CYAN}docker-gobuild ${GREEN}completed${RESET}"