Set access log format and Upstream

Set access log format to json_combined
Switch from $backend_variable to upstream
(Except for harbor, plane-minio, goaccess, opencloud)

data/conf.d/adminer.conf

@@ -1,4 +1,16 @@
-# Redirect HTTP to HTTPS
+upstream adminer_backend {
+    server adminer:8080;
+
+    # Keep up to 32 idle connections per worker
+    keepalive 16;
+
+    # Maximum time a connection can be idle
+    keepalive_timeout 60s;
+
+    # Maximum requests per keepalive connection
+    keepalive_requests 100;
+}
+
 server {
     listen 80;
     listen [::]:80;
@@ -62,7 +74,7 @@ server {
     # ssl_dhparam /etc/nginx/ssl/dhparam.pem;
 
     # Logging
-    access_log /var/log/nginx/adminer.novicelab.io_access.log VCOMBINED;
+    access_log /var/log/nginx/adminer.novicelab.io_access.log json_combined;
     error_log /var/log/nginx/adminer.novicelab.io_error.log debug;
 
     # Root and index
@@ -70,13 +82,13 @@ server {
     # index index.html index.htm;
 
     # include /etc/letsencrypt/options-ssl-nginx.conf;
-    set $adminer_backend adminer:8080; 
+    # set $adminer_backend adminer:8080; 
 
     location / { #/adminer {
         # rewrite ^/adminer/(.*)$ /$1 break;
 
         # proxy_pass http://10.0.0.251:9080/;
-        proxy_pass http://$adminer_backend;
+        proxy_pass http://adminer_backend;
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

data/conf.d/auth.conf

@@ -1,3 +1,16 @@
+upstream keycloak_backend {
+    server keycloak:80; 
+
+    # Keep up to 32 idle connections per worker
+    keepalive 16;
+
+    # Maximum time a connection can be idle
+    keepalive_timeout 60s;
+
+    # Maximum requests per keepalive connection
+    keepalive_requests 100;
+}
+
 # Redirect HTTP to HTTPS
 server {
     listen 80;
@@ -62,7 +75,7 @@ server {
     # ssl_dhparam /etc/nginx/ssl/dhparam.pem;
 
     # Logging
-    access_log /var/log/nginx/auth.novicelab.io_access.log VCOMBINED;
+    access_log /var/log/nginx/auth.novicelab.io_access.log json_combined;
     error_log /var/log/nginx/auth.novicelab.io_error.log debug;
 
     # Root and index
@@ -71,12 +84,12 @@ server {
 
     # include /etc/letsencrypt/options-ssl-nginx.conf;
     
-    set $keycloak_backend keycloak:80; 
+    # set $keycloak_backend keycloak:80; 
 
     # client_max_body_size 0;
         location / {
         # proxy_pass http://10.0.0.253:8085/auth/;
-        proxy_pass http://$keycloak_backend;
+        proxy_pass http://keycloak_backend;
             
 
         proxy_set_header Host $host;

data/conf.d/book.conf

@@ -1,3 +1,17 @@
+upstream bookstack_backend {
+    # server 10.0.0.251:6875/;
+    server bookstack:80;
+
+    # Keep up to 32 idle connections per worker
+    keepalive 16;
+
+    # Maximum time a connection can be idle
+    keepalive_timeout 60s;
+
+    # Maximum requests per keepalive connection
+    keepalive_requests 100;
+}
+
 # # Redirect HTTP to HTTPS
 server {
     listen 80;
@@ -62,7 +76,7 @@ server {
     # ssl_dhparam /etc/nginx/ssl/dhparam.pem;
 
     # Logging
-    access_log /var/log/nginx/book.novicelab.io_access.log VCOMBINED;
+    access_log /var/log/nginx/book.novicelab.io_access.log json_combined;
     error_log /var/log/nginx/book.novicelab.io_error.log debug;
 
     # Root and index
@@ -71,14 +85,14 @@ server {
 
     # include /etc/letsencrypt/options-ssl-nginx.conf;
     
-    set $bookstack_backend bookstack:80;
+    # set $bookstack_backend bookstack:80;
 
     # client_max_body_size 0;
     # BookStack (/docs)
     location / {
         # rewrite ^/docs/(.*) /$1 break;
-        # proxy_pass http://$bookstack_backend;
+        proxy_pass http://bookstack_backend;
-        proxy_pass http://10.0.0.251:6875/;
+        # proxy_pass http://10.0.0.251:6875/;
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

data/conf.d/drone.conf

@@ -1,3 +1,16 @@
+upstream drone_backend {
+    server drone:80;
+
+    # Keep up to 32 idle connections per worker
+    keepalive 16;
+
+    # Maximum time a connection can be idle
+    keepalive_timeout 60s;
+
+    # Maximum requests per keepalive connection
+    keepalive_requests 100;
+}
+
 server {
     listen 80;
     server_name drone.novicelab.io;
@@ -28,11 +41,11 @@ server {
     resolver_timeout 5s;
 
     # set $couch_backend 10.0.0.251:9001;
-    set $drone_backend drone:80;
+    # set $drone_backend drone:80;
-    set $drone_runner_backend drone-runner-1:3000;
+    # set $drone_runner_backend drone-runner-1:3000;
 
     location / {
-        proxy_pass http://$drone_backend;
+        proxy_pass http://drone_backend;
         proxy_set_header Host $http_host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

data/conf.d/gitea.conf

@@ -1,3 +1,16 @@
+upstream gitea_backend {
+    server gitea:3000; 
+
+    # Keep up to 32 idle connections per worker
+    keepalive 16;
+
+    # Maximum time a connection can be idle
+    keepalive_timeout 60s;
+
+    # Maximum requests per keepalive connection
+    keepalive_requests 100;
+}
+
 server {
     listen 443 ssl; #http2;
     listen [::]:443 ssl; # http2;
@@ -31,13 +44,13 @@ server {
     add_header Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline' 'unsafe-eval';" always;
 
     # Logging
-    access_log /var/log/nginx/gitea.novicelab.io_access.log VCOMBINED;
+    access_log /var/log/nginx/gitea.novicelab.io_access.log json_combined;
     error_log /var/log/nginx/gitea.novicelab.io_error.log debug;
 
-    set $gitea_backend gitea:3000; 
+    # set $gitea_backend gitea:3000; 
 
     location / {
-        proxy_pass http://$gitea_backend;
+        proxy_pass http://gitea_backend;
         proxy_set_header Host $http_host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

data/conf.d/goaccess.conf

@@ -1,3 +1,16 @@
+# upstream goaccess_backend {
+#     server goaccess:7890; 
+# 
+#     # Keep up to 32 idle connections per worker
+#     keepalive 16;
+# 
+#     # Maximum time a connection can be idle
+#     keepalive_timeout 60s;
+# 
+#     # Maximum requests per keepalive connection
+#     keepalive_requests 100;
+# }
+
 server {
     listen 80;
     listen [::]:80;
@@ -41,7 +54,7 @@ server {
     resolver_timeout 5s;
 
         # Logging
-    access_log /var/log/nginx/goaccess.novicelab.io_access.log VCOMBINED;
+    access_log /var/log/nginx/goaccess.novicelab.io_access.log json_combined;
     error_log /var/log/nginx/goaccess.novicelab.io_error.log debug;
 
     set $goaccess_backend goaccess:7890; 

data/conf.d/harbor.conf

@@ -1,3 +1,16 @@
+# upstream harbor_backend {
+#     server nginx-harbor:80; 
+# 
+#     # Keep up to 32 idle connections per worker
+#     keepalive 16;
+# 
+#     # Maximum time a connection can be idle
+#     keepalive_timeout 60s;
+# 
+#     # Maximum requests per keepalive connection
+#     keepalive_requests 100;
+# }
+
 server {
     listen 443 ssl; #http2;
     listen [::]:443 ssl; # http2;
@@ -31,7 +44,7 @@ server {
     add_header Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline' 'unsafe-eval';" always;
 
     # Logging
-    access_log /var/log/nginx/harbor.novicelab.io_access.log VCOMBINED;
+    access_log /var/log/nginx/harbor.novicelab.io_access.log json_combined;
     error_log /var/log/nginx/harbor.novicelab.io_error.log debug;
 
     # set $harbor_backend 10.0.0.251:9090; 

data/conf.d/hugo.conf

@@ -1,3 +1,16 @@
+upstream hugo_backend {
+    server hugo:1313;
+
+    # Keep up to 32 idle connections per worker
+    keepalive 16;
+
+    # Maximum time a connection can be idle
+    keepalive_timeout 60s;
+
+    # Maximum requests per keepalive connection
+    keepalive_requests 100;
+}
+
 # # Redirect HTTP to HTTPS
 server {
     listen 80;
@@ -62,7 +75,7 @@ server {
     # ssl_dhparam /etc/nginx/ssl/dhparam.pem;
 
     # Logging
-    access_log /var/log/nginx/novicelab.io_access.log VCOMBINED;
+    access_log /var/log/nginx/novicelab.io_access.log json_combined;
     error_log /var/log/nginx/novicelab.io_error.log debug;
 
     # Root and index
@@ -73,11 +86,11 @@ server {
     # include /etc/nginx/cloudflare-ips.conf;
 
     # include /etc/letsencrypt/options-ssl-nginx.conf;
-    set $hugo_backend hugo:1313; 
+    # set $hugo_backend hugo:1313; 
 
     location / {
         # proxy_pass http://10.0.0.251:9200/;
-        proxy_pass http://$hugo_backend;
+        proxy_pass http://hugo_backend;
         
         proxy_set_header Host $host;
         # proxy_set_header X-Real-IP $remote_addr;

data/conf.d/minio.conf

@@ -1,3 +1,29 @@
+upstream minio_backend {
+    server minio:9001;
+
+    # Keep up to 32 idle connections per worker
+    keepalive 16;
+
+    # Maximum time a connection can be idle
+    keepalive_timeout 60s;
+
+    # Maximum requests per keepalive connection
+    keepalive_requests 100;
+}
+
+upstream s3_backend {
+    server minio:9000; 
+
+    # Keep up to 32 idle connections per worker
+    keepalive 16;
+
+    # Maximum time a connection can be idle
+    keepalive_timeout 60s;
+
+    # Maximum requests per keepalive connection
+    keepalive_requests 100;
+}
+
 server {
     listen 80;
     server_name minio.novicelab.io; 
@@ -41,18 +67,18 @@ server {
     add_header Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline' 'unsafe-eval';" always;
 
     # Logging
-    access_log /var/log/nginx/minio.novicelab.io_access.log VCOMBINED;
+    access_log /var/log/nginx/minio.novicelab.io_access.log json_combined;
     error_log /var/log/nginx/minio.novicelab.io_error.log debug;
 
     # resolver 127.0.0.11 valid=30s;
-    set $minio_backend minio:9001; 
+    # set $minio_backend minio:9001; 
 
     # if ($http_x_forwarded_proto != "https") {
     #     return 301 https://$host$request_uri;
     # }
 
     location / {
-        proxy_pass http://$minio_backend;
+        proxy_pass http://minio_backend;
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -74,6 +100,12 @@ server {
     }
 }
 
+server {
+    listen 80;
+    server_name s3.novicelab.io; 
+    return 301 https://$host$request_uri; # Redirect HTTP to HTTPS
+}
+
 server {
     listen 443 ssl; #http2;
     listen [::]:443 ssl; # http2;
@@ -107,14 +139,14 @@ server {
     add_header Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline' 'unsafe-eval';" always;
 
     # Logging
-    access_log /var/log/nginx/s3.novicelab.io_access.log VCOMBINED;
+    access_log /var/log/nginx/s3.novicelab.io_access.log json_combined;
     error_log /var/log/nginx/s3.novicelab.io_error.log debug;
 
     # resolver 127.0.0.11 valid=30s;
-    set $s3_backend minio:9000; 
+    # set $s3_backend minio:9000; 
 
     location / {
-        proxy_pass http://$s3_backend;
+        proxy_pass http://s3_backend;
         proxy_set_header Host $http_host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

data/conf.d/opencloud.conf

@@ -1,3 +1,16 @@
+upstream opencloud_backend {
+    server 10.0.0.251:9200;
+
+    # Keep up to 32 idle connections per worker
+    keepalive 16;
+
+    # Maximum time a connection can be idle
+    keepalive_timeout 60s;
+
+    # Maximum requests per keepalive connection
+    keepalive_requests 100;
+}
+
 server {
     listen 80;
     server_name opencloud.novicelab.io;
@@ -15,7 +28,7 @@ server {
     ssl_session_cache   shared:SSL:10m;
 
     # Logging
-    access_log /var/log/nginx/opencloud.novicelab.io_access.log VCOMBINED;
+    access_log /var/log/nginx/opencloud.novicelab.io_access.log json_combined;
     error_log /var/log/nginx/opencloud.novicelab.io_error.log debug;
 
     # Security headers
@@ -47,8 +60,8 @@ server {
 
     location / {
         # Pass all other requests to CouchDB
-        proxy_pass              http://10.0.0.251:9200;
+        proxy_pass              http://10.0.0.250:9200;
-        #proxy_pass         http://$opencloud_backend/;
+        # proxy_pass         http://opencloud_backend/;
 
         proxy_set_header        Host $host;
         proxy_set_header        X-Real-IP $remote_addr;

data/conf.d/plane.conf

@@ -1,3 +1,41 @@
+upstream backend_web {
+    server plane-web:3000;
+    keepalive 16;
+    keepalive_timeout 60s;
+    keepalive_requests 100;
+}
+upstream backend_space {
+    server plane-space:3000;
+    keepalive 16;
+    keepalive_timeout 60s;
+    keepalive_requests 100;
+}
+upstream backend_admin {
+    server plane-admin:3000;
+    keepalive 16;
+    keepalive_timeout 60s;
+    keepalive_requests 100;
+}
+upstream backend_live {
+    server plane-live:3000;
+    keepalive 16;
+    keepalive_timeout 60s;
+    keepalive_requests 100;
+}
+upstream backend_api {
+    server plane-api:8000;
+    keepalive 16;
+    keepalive_timeout 60s;
+    keepalive_requests 100;
+}
+# upstream backend_minio {
+#     server minio:9000;
+#     keepalive 16;
+#     keepalive_timeout 60s;
+#     keepalive_requests 100;
+# }
+
+
 server {
     if ($host = plane.novicelab.io) {
         return 301 https://$host$request_uri;
@@ -40,16 +78,16 @@ server {
     add_header Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline' 'unsafe-eval';" always;
 
     # Logging
-    access_log /var/log/nginx/plane.novicelab.io_access.log VCOMBINED;
+    access_log /var/log/nginx/plane.novicelab.io_access.log json_combined;
     error_log /var/log/nginx/plane.novicelab.io_error.log debug;
 
     # resolver 127.0.0.11 valid=30s;
     # set $plane_backend 10.0.0.251:9020; 
-    set $backend_web    plane-web:3000;
+    # set $backend_web    plane-web:3000;
-    set $backend_space  plane-space:3000;
+    # set $backend_space  plane-space:3000;
-    set $backend_admin  plane-admin:3000;
+    # set $backend_admin  plane-admin:3000;
-    set $backend_live   plane-live:3000;
+    # set $backend_live   plane-live:3000;
-    set $backend_api    plane-api:8000;
+    # set $backend_api    plane-api:8000;
     set $backend_minio  minio:9000;
 
 
@@ -68,7 +106,7 @@ server {
         return 301 /spaces/;
     }
     location /spaces/ {
-        proxy_pass http://$backend_space;
+        proxy_pass http://backend_space;
 
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
@@ -81,7 +119,7 @@ server {
         return 301 /god-mode/;
     }
     location /god-mode/ {
-        proxy_pass http://$backend_admin;
+        proxy_pass http://backend_admin;
 
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
@@ -91,7 +129,7 @@ server {
 
     # Live
     location /live/ {
-        proxy_pass http://$backend_live;
+        proxy_pass http://backend_live;
 
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
@@ -101,7 +139,7 @@ server {
 
     # API & Auth
     location /api/ {
-        proxy_pass http://$backend_api;
+        proxy_pass http://backend_api;
 
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
@@ -109,7 +147,7 @@ server {
         proxy_set_header X-Forwarded-Proto $scheme;
     }
     location /auth/ {
-        proxy_pass http://$backend_api;
+        proxy_pass http://backend_api;
 
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
@@ -157,7 +195,7 @@ server {
 
     # Web (Default catch-all)
     location / {
-        proxy_pass http://$backend_web;
+        proxy_pass http://backend_web;
 
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;

data/conf.d/umami.conf

@@ -1,3 +1,16 @@
+upstream umami_backend {
+    server umami:3000;
+
+    # Keep up to 32 idle connections per worker
+    keepalive 16;
+
+    # Maximum time a connection can be idle
+    keepalive_timeout 60s;
+
+    # Maximum requests per keepalive connection
+    keepalive_requests 100;
+}
+
 # # Redirect HTTP to HTTPS
 server {
     listen 80;
@@ -62,7 +75,7 @@ server {
     # ssl_dhparam /etc/nginx/ssl/dhparam.pem;
 
     # Logging
-    access_log /var/log/nginx/umami.novicelab.io_access.log VCOMBINED;
+    access_log /var/log/nginx/umami.novicelab.io_access.log json_combined;
     error_log /var/log/nginx/umami.novicelab.io_error.log debug;
 
     # Root and index
@@ -70,11 +83,12 @@ server {
     # index index.html index.htm;
 
     # include /etc/letsencrypt/options-ssl-nginx.conf;
-    set $umami_backend umami:3000; 
+    # set $umami_backend umami:3000; 
 
     location / {
         # proxy_pass http://10.0.0.251:9200/;
-        proxy_pass http://$umami_backend;
+        # proxy_pass http://$umami_backend;
+        proxy_pass http://umami_backend;
         
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
@@ -90,7 +104,8 @@ server {
     }
         # 1. Allow public access to tracking scripts
     location ~ ^/(script\.js|umami\.js)$ {
-        proxy_pass http://$umami_backend;
+        # proxy_pass http://$umami_backend;
+        proxy_pass http://umami_backend;
         
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
@@ -107,7 +122,8 @@ server {
 
     # 2. Allow public access to tracking API (metrics collection)
     location /api/send {
-        proxy_pass http://$umami_backend;
+        # proxy_pass http://$umami_backend;
+        proxy_pass http://umami_backend;
         
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;

data/conf.d/vault.conf

@@ -1,3 +1,32 @@
+upstream vault_backend {
+    server 10.0.0.250:8090;
+
+    # Keep up to 32 idle connections per worker
+    keepalive 16;
+
+    # Maximum time a connection can be idle
+    keepalive_timeout 60s;
+
+    # Maximum requests per keepalive connection
+    keepalive_requests 100;
+}
+
+# # Redirect HTTP to HTTPS
+server {
+    listen 80;
+    listen [::]:80;
+    server_name vault.novicelab.io;
+    
+    # ACME challenge for Let's Encrypt certificate renewal
+    location /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+    }
+    
+    location / {
+        return 301 https://$server_name$request_uri;
+    }
+}
+
 server {
     listen 443 ssl; #http2;
     listen [::]:443 ssl; # http2;
@@ -31,14 +60,14 @@ server {
     add_header Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline' 'unsafe-eval';" always;
 
     # Logging
-    access_log /var/log/nginx/vault.novicelab.io_access.log VCOMBINED;
+    access_log /var/log/nginx/vault.novicelab.io_access.log json_combined;
     error_log /var/log/nginx/vault.novicelab.io_error.log debug;
-    set $vault_backend vaultwarden:443;
+    # set $vault_backend vaultwarden:443;
 
     location / {
         # proxy_pass http://$vault_backend;
-        # proxy_pass https://10.0.0.251:8100;
+        # proxy_pass http://10.0.0.250:8090;
-        proxy_pass http://10.0.0.250:8090;
+        proxy_pass http://vault_backend;
         proxy_set_header Host $http_host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

data/nginx.conf

@@ -23,30 +23,30 @@ http {
     resolver 8.8.8.8 valid=30s ipv6=off;
     resolver_timeout 11s;
 
-    # log_format json_combined escape=json '{'
+    log_format json_combined escape=json '{'
-    #     '"method":"$request_method",'
+        '"method":"$request_method",'
-    #     '"scheme":"$scheme",'
+        '"scheme":"$scheme",'
-    #     '"domain":"$host",'
+        '"domain":"$host",'
-    #     '"uri":"$request_uri",'
+        '"uri":"$request_uri",'
-    #     '"query_string":"$query_string",'
+        '"query_string":"$query_string",'
-    #     '"referer":"$http_referer",'
+        '"referer":"$http_referer",'
-    #     '"content_type":"$sent_http_content_type",'
+        '"content_type":"$sent_http_content_type",'
-    #     '"status": $status,'
+        '"status": $status,'
-    #     '"bytes_sent":$body_bytes_sent,'
+        '"bytes_sent":$body_bytes_sent,'
-    #     '"request_time":$request_time,'
+        '"request_time":$request_time,'
-    #     '"user_agent":"$http_user_agent",'
+        '"user_agent":"$http_user_agent",'
-    #     '"cache":"$upstream_cache_status",'
+        '"cache":"$upstream_cache_status",'
-    #     '"upstream_time": "$upstream_response_time",'
+        '"upstream_time": "$upstream_response_time",'
-    #     '"timestamp":"$time_iso8601",'
+        '"timestamp":"$time_iso8601",'
-    #     '"ip":"$http_x_forwarded_for"'
+        '"ip":"$http_x_forwarded_for"'
-    # '}';
+    '}';
-    log_format VCOMBINED '$host:$server_port '
+    # log_format VCOMBINED '$host:$server_port '
-                     '$remote_addr $remote_user [$time_local] '
+    #                  '$remote_addr $remote_user [$time_local] '
-                     '"$request" $status $body_bytes_sent '
+    #                  '"$request" $status $body_bytes_sent '
-                     '"$http_referer" "$http_user_agent"';
+    #                  '"$http_referer" "$http_user_agent"';
 
 
-    access_log  /var/log/nginx/access.log VCOMBINED;
+    access_log  /var/log/nginx/access.log json_combined;
     error_log   /var/log/nginx/error.log debug;
     
     sendfile on;
@@ -64,9 +64,9 @@ http {
     large_client_header_buffers 4 32k;
 
     # If using Nginx as a proxy to the Harbor core/registry
-    proxy_buffer_size 16k;
+    proxy_buffer_size          128k;
-    proxy_buffers 4 32k;
+    proxy_buffers              4 256k;
-    proxy_busy_buffers_size 64k;
+    proxy_busy_buffers_size    256k;
 
     proxy_connect_timeout 300;
     proxy_send_timeout 300;