# Change YOUR_ACCESS_KEY to your Access Key
# Change example.com (server_name) to your website url
# Change /path/to/your/root to the correct value

worker_processes  1;

events {
    worker_connections  1024;
}

http {
    include       mime.types;
    default_type  application/octet-stream;

    sendfile           on;
    keepalive_timeout  65;

    map $http_user_agent $spider_ua {
        default       0;
        "~*SPIDER" 0;

        "~*googlebot"                               1;
        "~*yahoo!\ slurp"                           1;
        "~*bingbot"                                 1;
        "~*yandex"                                  1;
        "~*baiduspider"                             1;
        "~*facebookexternalhit"                     1;
        "~*twitterbot"                              1;
        "~*rogerbot"                                1;
        "~*linkedinbot"                             1;
        "~*embedly"                                 1;
        "~*quora\ link\ preview"                    1;
        "~*showyoubot"                              1;
        "~*outbrain"                                1;
        "~*pinterest\/0\."                          1;
        "~*developers.google.com\/\+\/web\/snippet" 1;
        "~*slackbot"                                1;
        "~*vkshare"                                 1;
        "~*w3c_validator"                           1;
        "~*redditbot"                               1;
        "~*applebot"                                1;
        "~*whatsapp"                                1;
        "~*flipboard"                               1;
        "~*tumblr"                                  1;
        "~*bitlybot"                                1;
        "~*skypeuripreview"                         1;
        "~*nuzzel"                                  1;
        "~*discordbot"                              1;
        "~*google\ page\ speed"                     1;
        "~*qwantify"                                1;
        "~*pinterestbot"                            1;
        "~*bitrix\ link\ preview"                   1;
        "~*xing-contenttabreceiver"                 1;
        "~*chrome-lighthouse"                       1;
        "~*telegrambot"                             1;
        "~*google-inspectiontool"                   1;
        "~*petalbot"                                1;
        "~*pagespeed\ insights"                     1;
        "~*lighthouse"                              1;
        "~*headlesschrome"                          1;
        "~*web\ rendering\ service"                 1;
        "~*google\ page\ speed\ insights"           1;
        "~*webpagetest"                             1;
        "~*gtmetrix"                                1;
        "~*pingdom"                                 1;
        "~*sitespeed.io"                            1;
        "~*speedcurve"                              1;
        "~*uptimerobot"                             1;
        "~*statuscake"                              1;
        "~*browserstack"                            1;
        "~*dotcom-monitor"                          1;
        "~*keycdn"                                  1;
        "~*site24x7"                                1;
        "~*k6"                                      1;
        "~*calibreapp"                              1;
        "~*paessler"                                1;
    }

    map $args $spider_args {
        default $spider_ua;
        "~(^|&)_escaped_fragment_=" 1;
    }

    map $http_x_spider $x_spider {
        default $spider_args;
        "1"     0;
    }

    map $uri $spider {
        default $x_spider;
        "~*\.(js|css|xml|less|png|jpg|jpeg|gif|pdf|txt|ico|rss|zip|mp3|rar|exe|wmv|doc|avi|ppt|mpg|mpeg|tif|wav|mov|psd|ai|xls|mp4|m4a|swf|dat|dmg|iso|flv|m4v|torrent|ttf|woff|woff2|svg|eot)" 0;
    }

    server {
        listen       80;
        server_name  example.com;
        root   /path/to/your/root;

        location / {

            if ($spider = 1) {
                rewrite (.*) /spider last;
            }
            try_files $uri /index.html = 404;
        }

        location /spider {
            if ($spider = 0) {
                return 404;
            }

            proxy_set_header x-tts-cc-access-key YOUR_ACCESS_KEY;
            proxy_set_header x-int-type Nginx;

            proxy_hide_header Cache-Control;
            add_header Cache-Control "private,max-age=600,must-revalidate";

            #resolve using Google's DNS server to force DNS resolution and prevent caching of IPs
            resolver 8.8.8.8 8.8.4.4;
            set $spider_host "prod.service.utility-server.com";
            #Should end with "/"
            proxy_pass https://$spider_host/; #With unique identifier, if its not a dmain https://$spider_host/identifier/
            rewrite .* /$scheme://$host$request_uri? break;
        }
    }
}