Workers transiently dropping REQUEST BODY sending empty body to ORIGIN Server.
We are using CF workers to chnange the hostname and direct to different origion servers based on some URL paths and query parameters. However we have seen less 1% 524 errors. COreeposnding CF RAY ids were not present in origin server. In order to understand the pattern and started to log both request and response contents. However ever since we did, we started getting calls from customers they are are saying invalidation errors(422), that is due to request content being empty. I am suspecting that this is due to REQUEST.CLONE() we did( it was not there before).
4 Replies
Code -
`export default {
async fetch(
request: Request,
env: Env,
ctx: ExecutionContext,
): Promise<Response> {
ctx.passThroughOnException();
const requestForContent = request.clone();
try {
const response = await fetch(request);
if (response.status >= 520 && response.status <= 526) {
const errorMessages: Record<number, string> = {
520: 'Unknown error from origin server',
521: 'Web server is down',
522: 'Connection timed out to origin',
523: 'Origin is unreachable',
524: 'Origin server timeout',
525: 'SSL handshake failed',
526: 'Invalid SSL certificate'
};
const requestContent = await getRequestContent(requestForContent);
// Clone response once for content
const responseForContent = response.clone();
const responseContent = await getResponseContent(responseForContent);
console.error({
application: 'XXXXXXXXXX',
level: 'error',
message: errorMessages[response.status] || 'Unknown origin error',
request_id: request.headers.get('cf-ray') || 'no-ray-id',
timestamp: new Date().toISOString(),
// Request details
// request_content_value: redactPII(requestContent),
// request_method: request.method,
// request_path: new URL(request.url).pathname,
// request_headers: sanitizeHeaders(request.headers),
// Response details
response_status: response.status,
// response_headers: sanitizeHeaders(response.headers),
// response_content_value: redactPII(responseContent)
});
};
`export default {
async fetch(
request: Request,
env: Env,
ctx: ExecutionContext,
): Promise<Response> {
ctx.passThroughOnException();
const requestForContent = request.clone();
try {
const response = await fetch(request);
if (response.status >= 520 && response.status <= 526) {
const errorMessages: Record<number, string> = {
520: 'Unknown error from origin server',
521: 'Web server is down',
522: 'Connection timed out to origin',
523: 'Origin is unreachable',
524: 'Origin server timeout',
525: 'SSL handshake failed',
526: 'Invalid SSL certificate'
};
const requestContent = await getRequestContent(requestForContent);
// Clone response once for content
const responseForContent = response.clone();
const responseContent = await getResponseContent(responseForContent);
console.error({
application: 'XXXXXXXXXX',
level: 'error',
message: errorMessages[response.status] || 'Unknown origin error',
request_id: request.headers.get('cf-ray') || 'no-ray-id',
timestamp: new Date().toISOString(),
// Request details
// request_content_value: redactPII(requestContent),
// request_method: request.method,
// request_path: new URL(request.url).pathname,
// request_headers: sanitizeHeaders(request.headers),
// Response details
response_status: response.status,
// response_headers: sanitizeHeaders(response.headers),
// response_content_value: redactPII(responseContent)
});
};
When did these errors start? Did you make any changes or has it always been this way?
Well for one, you aren't returning anything. Not really able to troubleshoot / understand whats happening without including the part of the code you are having troubles with. Request/Response bodies can only be read once, so if they are consumed by something and then passed to the user, it is gonna be empty.
Thanks @Erisa @downeydabber We were getting this erros before as well, but for fewer, after recent deployment with some changes. We started getting more of them.
This was code before uptick -
This is code which saw uptick. We added code cpature 524 error reasons where in we are cloing request.
export default {
async fetch(
request: Request,
env: Env,
ctx: ExecutionContext,
): Promise<Response> {
ctx.passThroughOnException();
const url = new URL(request.url);
const params = new URLSearchParams(url.search);
const regionalParam = params.get('regional');
const isRegional =
url.pathname.includes('/fasts/') &&
['y', '1', 'yes', 'true'].includes(regionalParam?.toLowerCase() || '');
if (!isRegional && url.host === env.CENTRAL_DOMAIN) {
// Not fast and no need to rewrite domain, pass it to the origin.
return fetch(request); // Pass the original request directly
}
if (isRegional) {
url.host = env.REGIONAL_DOMAIN || url.host;
} else {
url.host = env.CENTRAL_DOMAIN;
}
const headers = request.headers;
const method = request.method;
const baseParams = { headers, method };
const reqParams = ['get', 'head'].includes(request.method.toLowerCase())
? baseParams
: {
...baseParams,
body: await request.text(),
};
const newRequest = new Request(url, reqParams);
return fetch(newRequest);
},
};
export default {
async fetch(
request: Request,
env: Env,
ctx: ExecutionContext,
): Promise<Response> {
ctx.passThroughOnException();
const url = new URL(request.url);
const params = new URLSearchParams(url.search);
const regionalParam = params.get('regional');
const isRegional =
url.pathname.includes('/fasts/') &&
['y', '1', 'yes', 'true'].includes(regionalParam?.toLowerCase() || '');
if (!isRegional && url.host === env.CENTRAL_DOMAIN) {
// Not fast and no need to rewrite domain, pass it to the origin.
return fetch(request); // Pass the original request directly
}
if (isRegional) {
url.host = env.REGIONAL_DOMAIN || url.host;
} else {
url.host = env.CENTRAL_DOMAIN;
}
const headers = request.headers;
const method = request.method;
const baseParams = { headers, method };
const reqParams = ['get', 'head'].includes(request.method.toLowerCase())
? baseParams
: {
...baseParams,
body: await request.text(),
};
const newRequest = new Request(url, reqParams);
return fetch(newRequest);
},
};
export default {
async fetch(
request: Request,
env: Env,
ctx: ExecutionContext,
): Promise<Response> {
ctx.passThroughOnException();
const url = new URL(request.url);
const params = new URLSearchParams(url.search);
const regionalParam = params.get('regional');
const isRegional =
url.pathname.includes('/fasts/') &&
['y', '1', 'yes', 'true'].includes(regionalParam?.toLowerCase() || '');
if (!isRegional && url.host === env.CENTRAL_DOMAIN) {
// Not fast and no need to rewrite domain, pass it to the origin.
return await handleFetchWithTimeout(newRequest); // Pass the original request directly
}
if (isRegional) {
url.host = env.REGIONAL_DOMAIN || url.host;
} else {
url.host = env.CENTRAL_DOMAIN;
}
const headers = request.headers;
const method = request.method;
const baseParams = { headers, method };
const reqParams = ['get', 'head'].includes(request.method.toLowerCase())
? baseParams
: {
...baseParams,
body: await request.text(),
};
const newRequest = new Request(url, reqParams);
return await handleFetchWithTimeout(newRequest);
},
};
async function handleFetchWithTimeout(
request: Request,
): Promise<Response> {
ctx.passThroughOnException();
const requestForContent = request.clone();
try {
const response = await fetch(request);
if (response.status >= 520 && response.status <= 526) {
const errorMessages: Record<number, string> = {
520: 'Unknown error from origin server',
521: 'Web server is down',
522: 'Connection timed out to origin',
523: 'Origin is unreachable',
524: 'Origin server timeout',
525: 'SSL handshake failed',
526: 'Invalid SSL certificate',
};
const requestContent = await getRequestContent(requestForContent);
const responseForContent = response.clone();
const responseContent = await getResponseContent(responseForContent);
console.error({
application: 'XXXXXXXXXX',
level: 'error',
message: errorMessages[response.status] || 'Unknown origin error',
request_id: request.headers.get('cf-ray') || 'no-ray-id',
timestamp: new Date().toISOString(),
// Request details
// request_content_value: redactPII(requestContent),
// request_method: request.method,
// request_path: new URL(request.url).pathname,
// request_headers: sanitizeHeaders(request.headers),
// Response details
response_status: response.status,
// response_headers: sanitizeHeaders(response.headers),
// response_content_value: redactPII(responseContent),
});
}
return response; // Return the response, even if an error occurred
} catch (error) {
console.error({
application: 'XXXXXXXXXX',
level: 'error',
message: 'An unexpected error occurred',
error: error.message || String(error), // include the error message
request_id: request.headers.get('cf-ray') || 'no-ray-id',
timestamp: new Date().toISOString()
});
return new Response('Internal Server Error', { status: 500 }); // Return a 500 response on error
}
};
async function getRequestContent(request: Request): Promise<string> {
try {
// More error handling
return await request.text();
} catch (error) {
return 'Could not retrieve request body';
}
}
async function getResponseContent(response: Response): Promise<string> {
try {
// More error handling
return await response.text();
} catch (error) {
return 'Could not retrieve response body';
}
}
export default {
async fetch(
request: Request,
env: Env,
ctx: ExecutionContext,
): Promise<Response> {
ctx.passThroughOnException();
const url = new URL(request.url);
const params = new URLSearchParams(url.search);
const regionalParam = params.get('regional');
const isRegional =
url.pathname.includes('/fasts/') &&
['y', '1', 'yes', 'true'].includes(regionalParam?.toLowerCase() || '');
if (!isRegional && url.host === env.CENTRAL_DOMAIN) {
// Not fast and no need to rewrite domain, pass it to the origin.
return await handleFetchWithTimeout(newRequest); // Pass the original request directly
}
if (isRegional) {
url.host = env.REGIONAL_DOMAIN || url.host;
} else {
url.host = env.CENTRAL_DOMAIN;
}
const headers = request.headers;
const method = request.method;
const baseParams = { headers, method };
const reqParams = ['get', 'head'].includes(request.method.toLowerCase())
? baseParams
: {
...baseParams,
body: await request.text(),
};
const newRequest = new Request(url, reqParams);
return await handleFetchWithTimeout(newRequest);
},
};
async function handleFetchWithTimeout(
request: Request,
): Promise<Response> {
ctx.passThroughOnException();
const requestForContent = request.clone();
try {
const response = await fetch(request);
if (response.status >= 520 && response.status <= 526) {
const errorMessages: Record<number, string> = {
520: 'Unknown error from origin server',
521: 'Web server is down',
522: 'Connection timed out to origin',
523: 'Origin is unreachable',
524: 'Origin server timeout',
525: 'SSL handshake failed',
526: 'Invalid SSL certificate',
};
const requestContent = await getRequestContent(requestForContent);
const responseForContent = response.clone();
const responseContent = await getResponseContent(responseForContent);
console.error({
application: 'XXXXXXXXXX',
level: 'error',
message: errorMessages[response.status] || 'Unknown origin error',
request_id: request.headers.get('cf-ray') || 'no-ray-id',
timestamp: new Date().toISOString(),
// Request details
// request_content_value: redactPII(requestContent),
// request_method: request.method,
// request_path: new URL(request.url).pathname,
// request_headers: sanitizeHeaders(request.headers),
// Response details
response_status: response.status,
// response_headers: sanitizeHeaders(response.headers),
// response_content_value: redactPII(responseContent),
});
}
return response; // Return the response, even if an error occurred
} catch (error) {
console.error({
application: 'XXXXXXXXXX',
level: 'error',
message: 'An unexpected error occurred',
error: error.message || String(error), // include the error message
request_id: request.headers.get('cf-ray') || 'no-ray-id',
timestamp: new Date().toISOString()
});
return new Response('Internal Server Error', { status: 500 }); // Return a 500 response on error
}
};
async function getRequestContent(request: Request): Promise<string> {
try {
// More error handling
return await request.text();
} catch (error) {
return 'Could not retrieve request body';
}
}
async function getResponseContent(response: Response): Promise<string> {
try {
// More error handling
return await response.text();
} catch (error) {
return 'Could not retrieve response body';
}
}