The fix: decide with Hrefine, hand on href
Hrefine doesn't change how your HTTP client reads URLs. So check
the URL Hrefine parsed, then pass on its href, never
the original string:
- Parse once, with
Url.parse, which gives null for anything that isn't a URL.
- Check the parsed parts:
scheme, hostKind (a name, or an address however it's written), and hostname.
- Or use a policy, built once:
UrlPolicy.fetch, plus your allowlist, and its one check covers the scheme, credentials, private and loopback addresses however they're written, ports, a host allowlist, redirect targets, and strict parsing, and says which rule failed.
- Hand on
href, the standard's canonical form: no backslashes, tabs, or odd address forms left to disagree on. For clients that follow RFC 3986 strictly, such as Java's URI, hand on strictHref.
href has no credentials, by design, unlike JavaScript's URL.href: the username and password in the URL are left out, so handing it on or storing it can't leak them. Only to send them deliberately, use hrefWithCredentials(), and never log it. withoutCredentials() gives a Url without them.- Check redirects too, or turn them off. For a
next parameter, resolve it against your own site and check sameOrigin.
- Log
toString(): it masks credentials as ***, so https://user:pass@host/ logs as https://***:***@host/, showing there were some. Tokens in a query or fragment aren't masked, so don't log URLs that may carry them.
From the Python tour's “Using it safely” section, which CI runs in every language.
const std::set<std::string> allowed = {"example.com", "api.example.com"};
// The URL to fetch, or why not.
std::string check(std::string const &input) {
auto url = Url::parse(input);
if (!url) return "refuse: not a URL";
if (url->get_scheme() != "https") return "refuse: scheme " + url->get_scheme();
// An IP address isn't a name on the list, however it's written. Each
// kind is one object, so == works.
if (url->get_hostKind() != HostKind::domain()) {
return "refuse: " + url->get_hostKind()->toString() + " host " + url->get_hostname();
}
if (!allowed.count(url->get_hostname())) return "refuse: host " + url->get_hostname();
// With libcurl: curl_easy_setopt(curl, CURLOPT_URL, url->get_href().c_str()),
// never input, and leave CURLOPT_FOLLOWLOCATION off, or check each
// redirect the same way. A hostname doesn't say where DNS sends it.
return "fetch " + url->get_href();
}
var allowed = new HashSet<string> { "example.com", "api.example.com" };
// The URL to fetch, or why not.
string Check(string input)
{
var url = Url.Parse(input);
if (url is null) return "refuse: not a URL";
if (url.Scheme != "https") return $"refuse: scheme {url.Scheme}";
// An IP address isn't a name on the list, however it's written. Each
// kind is one object, so == works.
if (url.HostKind != HostKind.Domain) return $"refuse: {url.HostKind!.ToString()} host {url.Hostname}";
if (!allowed.Contains(url.Hostname)) return $"refuse: host {url.Hostname}";
// await client.GetAsync(url.Href), never input, with a client whose
// handler has AllowAutoRedirect = false, or check each redirect the
// same way. A hostname doesn't say where DNS sends it.
return $"fetch {url.Href}";
}
static final Set<String> ALLOWED = Set.of("example.com", "api.example.com");
/** The URL to fetch, or why not. */
static String check(String input) {
Url url = Url.parse(input);
if (url == null) return "refuse: not a URL";
if (!url.getScheme().equals("https")) return "refuse: scheme " + url.getScheme();
// An IP address isn't a name on the list, however it's written.
// Each kind is one object, so == works.
if (url.getHostKind() != HostKind.domain) {
return "refuse: " + url.getHostKind() + " host " + url.getHostname();
}
if (!ALLOWED.contains(url.getHostname())) return "refuse: host " + url.getHostname();
// client.send(HttpRequest.newBuilder(URI.create(url.getHref())).build(), ...),
// never input, with a client built with
// .followRedirects(HttpClient.Redirect.NEVER), or check each redirect
// the same way. URI.create follows RFC 3986, which rejects a few
// characters href leaves as they are, such as | and ^. A hostname
// doesn't say where DNS sends it.
return "fetch " + url.getHref();
}
const allowed = new Set(["example.com", "api.example.com"]);
// The URL to fetch, or why not.
function check(input) {
const url = Url.parse(input);
if (url === null) return "refuse: not a URL";
if (url.scheme !== "https") return `refuse: scheme ${url.scheme}`;
// An IP address isn't a name on the list, however it's written.
if (url.hostKind !== HostKind.domain) return `refuse: ${url.hostKind} host ${url.hostname}`;
if (!allowed.has(url.hostname)) return `refuse: host ${url.hostname}`;
// await fetch(url.href, { redirect: "manual" }), never input. Check each
// redirect the same way. A hostname doesn't say where DNS sends it.
return `fetch ${url.href}`;
}
local allowed = { ["example.com"] = true, ["api.example.com"] = true }
-- The URL to fetch, or why not.
local function check(input)
local url = null_to_nil(Url.parse(input))
if url == nil then return "refuse: not a URL" end
if url.scheme ~= "https" then return "refuse: scheme " .. url.scheme end
-- An IP address isn't a name on the list, however it's written. Each
-- kind is one table, so == works.
if url.hostKind ~= HostKind.domain then
return "refuse: " .. url.hostKind:toString() .. " host " .. url.hostname
end
if not allowed[url.hostname] then return "refuse: host " .. url.hostname end
-- With LuaSocket (and LuaSec for https): http.request{ url = url.href,
-- redirect = false, ... }, never input. Check each redirect the same
-- way. A hostname doesn't say where DNS sends it.
return "fetch " .. url.href
end
ALLOWED = {"example.com", "api.example.com"}
def check(text):
"""The URL to fetch, or why not."""
url = Url.parse(text)
if url is None:
return "refuse: not a URL"
if url.scheme != "https":
return f"refuse: scheme {url.scheme}"
# An IP address isn't a name on the list, however it's written.
if url.host_kind != HostKind.domain:
return f"refuse: {url.host_kind.to_string()} host {url.hostname}"
if url.hostname not in ALLOWED:
return f"refuse: host {url.hostname}"
# requests.get(url.href) or httpx.get(url.href), never text. Turn off
# redirects (allow_redirects=False, follow_redirects=False) or check
# each hop the same way. A hostname doesn't say where DNS sends it.
return f"fetch {url.href}"
const ALLOWED: [&str; 2] = ["example.com", "api.example.com"];
/// The URL to fetch, or why not.
fn check(input: &str) -> String {
let Some(url) = Url::parse(input, NO_BASE) else {
return "refuse: not a URL".into();
};
if url.scheme().as_str() != "https" {
return format!("refuse: scheme {}", url.scheme());
}
// An IP address isn't a name on the list, however it's written.
// HostKind has no PartialEq, but eq compares kinds.
match url.host_kind() {
Some(kind) if kind.eq(HostKind::domain()) => {}
Some(kind) => return format!("refuse: {} host {}", kind.to_string(), url.hostname()),
None => return "refuse: no host".into(),
}
if !ALLOWED.contains(&url.hostname().as_str()) {
return format!("refuse: host {}", url.hostname());
}
// reqwest::get(url.href().as_str()), never input, with a client built
// with .redirect(Policy::none()), or check each redirect the same way.
// A hostname doesn't say where DNS sends it.
format!("fetch {}", url.href())
}
Given href, or its strict form, the parsers agree
The same parsers, given each of the 62 recorded URLs as
typed, then given Hrefine's href for each URL Hrefine
accepts, and then its strictHref: href
with the few characters RFC 3986 doesn't allow (such as
| and {) escaped, for strict parsers.
Both are given with any credentials kept
(hrefWithCredentials() and
strictHrefWithCredentials()), so tricks in the user
info are tested too.
Counted is how often each reads the same host (or path, for path
tricks) as the browser.
Given href, the misses are paths: C# decodes escaped
dots (%2E%2E%2F), and a % without hex
digits (/%zz) and a |, which
href keeps as they are, are refused by Java's
URI, which follows RFC 3986, and by Go, or escaped
again by C# and Go. strictHref escapes them, and
decodes escaped dots, as RFC 3986's normalization does, so every
parser agrees but Java's URL and curl, which refuse
javascript:, a scheme they don't fetch, which is
fine. Browsers read each strict form as they read its
href, which the build checks.