1

I seem to be having trouble with Grafana behind an nginx proxy. I'm getting console errors and errors in Grafana, and the changes I try to make in Grafana are not working.

Errors: From within Grafana I get "Origin not allowed" when trying to access the site from https://grafana.domainname.dev and making any action.

I see console errors of

this kind whenenver i load a page: Failed to load resource: the server responded with a status of 4030(forbidden) URL: https://grafana.domainname.dev/public/fonts/roboto/RxZJdnzeo3R5zSexge8UUVtXRa8TVwTICgirnJhmVJw.woff2 Failed to load resource: the server responded with a status of 4030(forbidden) URL: https://grafana.domainname.dev/public/fonts/fontawesome-webfont.ttf?v=4.7.0

this kind whenever i go into submenus or try to make changes (these two errors occur the same time as origin not allowed error) Failed to load resource: the server responded with a status of 4030(forbidden) URL: https://grafana.domainname.dev/api/frontend-metrics

Uncaught (in promise)

{
    "status": 403,
    "statusText": "Forbidden",
    "data": {
        "message": "origin not allowed\n",
        "error": "Forbidden",
        "response": "origin not allowed\n"
    },
    "config": {
        "method": "POST",
        "url": "api/frontend-metrics",
        "data": {
            "events": [
                {
                    "name": "frontend_boot_first-paint_time_seconds",
                    "value": 1.045
                },
                {
                    "name": "frontend_boot_first-contentful-paint_time_seconds",
                    "value": 1.045
                },
                {
                    "name": "frontend_boot_load_time_seconds",
                    "value": 1.311
                }
            ]
        },
        "retry": 0,
        "headers": {
            "X-Grafana-Org-Id": 1
        },
        "hideFromInspector": true
    }
}

Grafana Config:

General Configuration:

instance_name = grafana.domainname.dev

Server block: 
[server]
# Protocol (http, https, h2, socket)
;protocol = http

# The ip address to bind to, empty will bind to all interfaces
;http_addr =

# The http port  to use
;http_port = 3000

# The public facing domain name used to access grafana from a browser
domain = grafana.domainname.dev

# Redirect to correct domain if host header does not match domain
# Prevents DNS rebinding attacks
;enforce_domain = true

# The full public facing url you use in browser, used for redirects and emails
# If you use reverse proxy and sub path specify full url (with sub path)
root_url = https://grafana.domainname.dev/

# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons.
;serve_from_sub_path = false

Everything else is unaltered, except for mail settings I set further down. When i try to turn on enforce_domain it ends up in a redirect loop and the browser can't connect.

NGINX Config:

/etc/nginx/sites-available/grafana.domainname.dev :

map $http_upgrade $connection_upgrade {
  default upgrade;
  '' close;
}

server {

        root /var/www/grafana/html;
        index index.html index.htm index.nginx-debian.html;

        server_name grafana.domainname.dev www.grafana.domainname.dev;

        location / {
                proxy_pass http://localhost:3000/;
        }

        location /api/live {
          rewrite  ^/(.*)  /$1 break;
          proxy_http_version 1.1;
          proxy_set_header Upgrade $http_upgrade;
          proxy_set_header Connection $connection_upgrade;
          proxy_set_header Host $http_host;
          proxy_pass http://localhost:3000/;
        }




    listen [::]:443 ssl ipv6only=on; # managed by Certbot
    listen 443 ssl; # managed by Certbot
    ssl_certificate /etc/letsencrypt/live/grafana.domainname.dev/fullchain.pem; # managed by Certbot
    ssl_certificate_key /etc/letsencrypt/live/grafana.domainname.dev/privkey.pem; # managed by Certbot
    include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot

}
server {
    if ($host = grafana.domainname.dev) {
        return 301 https://$host$request_uri;
    } # managed by Certbot


        listen 80;
        listen [::]:80;

        server_name grafana.domainname.dev www.grafana.domainname.dev;
    return 404; # managed by Certbot


}

Some of the above was added by certbot, and I didn't touch any of that, and I tried to follow grafana's tutorial as well as some forum suggestions but to no avail!

/etc/nginx/nginx.conf :

user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;

events {
        worker_connections 768;
        # multi_accept on;
}

http {
        map $http_upgrade $connection_upgrade {
         default upgrade;
         '' close;
        }


        ##
        # Basic Settings
        ##

        sendfile on;
        tcp_nopush on;
        tcp_nodelay on;
        keepalive_timeout 65;
        types_hash_max_size 2048;
        # server_tokens off;

        # server_names_hash_bucket_size 64;
        # server_name_in_redirect off;

        include /etc/nginx/mime.types;
        default_type application/octet-stream;

        ##
        # SSL Settings
        ##

        ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE
        ssl_prefer_server_ciphers on;

        ##
        # Logging Settings
        ##

        access_log /var/log/nginx/access.log;
        error_log /var/log/nginx/error.log;

        ##
        # Gzip Settings
        ##

        gzip on;

        # gzip_vary on;
        # gzip_proxied any;
        # gzip_comp_level 6;
        # gzip_buffers 16 8k;
        # gzip_http_version 1.1;
        # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;

        ##
        # Virtual Host Configs
        ##

        include /etc/nginx/conf.d/*.conf;
        include /etc/nginx/sites-enabled/*;
}

This is all standard except I did add the map $http_upgrade part however that doesn't look to have done anything.

If anyone has any insight here, I'd really appreciate it! It looks like something isn't making it back to localhost:3000 however I'm not sure why/how that is possible when nginx is configured to send everything from grafana.domainname.dev to localhost:3000. Appreciate the help in advance!

1 Answer 1

1

The Host header needs to be preserved now.

For nginx: proxy_set_header Host $http_host; in BOTH your Location blocks.

For Apache2: ProxyPreserveHost On [1]

[1] https://github.com/grafana/grafana/issues/45117#issuecomment-1033842787

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.