Cloudflare

note

Double check the user agents whitelisted for every deployment, since you may not need all of them, or you might want to add some

Every deployment can also be used as a Edge Cache API. You can simply forward requests to us when you detect a bot in a user agent.

This is the recommended way to deploy.

Seeing a WRCacheHit header with a value of 1 indicates the cache was hit - but you can safely forward any requests since we will simply fetch origin if it's not in the cache. If it's from a crawler that doesn't support JS we'll pre-render it, if the the client does support JS it'll be rendered on the fly.

Cloudflare Workers

We recommend using WebRender in combination with Cloudflare Workers for the best results. For optimal speed, consider taking advantage of their KV Storage solution so you can minimize requests to the WebRender API even further and save costs.

Step 1: Create a Cloudflare Worker

Step 2: Go to Workers -> Add route and use the worker on your domain

img

Step 3: Paste the code below into your worker

//replace example.com with your site name and YOUR_DEPLOYMENT with your deployment id
//https://perishablepress.com/list-all-user-agents-top-search-engines/
const triggers = ["ahrefs", "serp", "chrome-lighthouse","headlesschrome", "aolbuild", "google", "bingbot", "bingpreview", "msnbot", "duckduckgo", "teoma", "slurp", "yandex", "baidu"]
async function handleRequest(request) {
// don't even check for statics
if(isAssetRequest(request.url)) {
return fetch(request);
}
let hasHttps = request.url.indexOf("https") !== -1;
let url = new URL(request.url.replace((hasHttps ? "https://example.com" : "http://example.com"), "https://YOUR_DEPLOYMENT.webrender.io"));
const newRequest = new Request(url, {headers: request.headers,
body: request.headers.get('content-length') === "0" ? undefined : request.body})
try {
let trigger = false;
for (let i = 0; i < triggers.length; i++) {
if (triggers &&
request.headers.get('user-agent') &&
request.headers.get('user-agent').toLowerCase().match(triggers[i])) {
trigger = true;
}
}
if(trigger === false) {
return fetch(request);
}
const c = await fetch(newRequest, { cf: { cacheTtl: 86400 } } );
if(c.status == 200) {
return c;
} else {
return fetch(request);
}
} catch (e) {
return new Response(JSON.stringify({ error: e.message }), { status: 500 })
}
}
function isAssetRequest(path) {
return path.match('.mp4|.html|.webm|.webm|.avi|.ico|.svg|.woff|.woff|.woff2|.gif|.png|.jpg|.json|.js|.css|.bundle|.json|.xml');
}
addEventListener('fetch', event => {
event.respondWith(handleRequest(event.request))
})