Inside unjs/ofetch query search params

Sat May 25 2024•8 min read

unjs ofetch ufo security prototype pollution

參考版本：

ofetch v1.3.4
ufo v1.5.3

紀錄 ofetch 如何處理 options.query

ofetch('/movie?lang=en', {
  query: { id: 123 },
});
// url: "/movie?lang=en&id=123"

上方 usage，對應到 ofetch 的實作細節如下

ofetch

import { withQuery } from "ufo";
 
const $fetchRaw: $Fetch["raw"] = async function $fetchRaw(
  _request,
  _options = {}
) {
  const context: FetchContext = {
    request: _request,
    options: mergeFetchOptions(_options, globalOptions.defaults, Headers),
    response: undefined,
    error: undefined,
  };
  // skip
  if (typeof context.request === "string") {
    if (context.options.baseURL) {
      context.request = withBase(context.request, context.options.baseURL);
    }
    if (context.options.query) {
      context.request = withQuery(context.request, context.options.query);
    }
  }
  // skip
}

withQuery 是由多個不同 function 組合而成，這裡先做個小結，後面會對這些 function 逐一進行拆解

const parsed = parseURL(input) 是將字串格式，轉換成帶有 URL 資料的物件 (query, pathname, etc.)
const mergedQuery = { ...parseQuery(parsed.search), ...query }
1. 先將 parsed.search 字串格式，轉換成 key value pair 的物件格式
2. 以新增或取代的方式，合併兩個 query params 物件
parsed.search = stringifyQuery(mergedQuery)
1. 將物件格式的 mergedQuery 轉換成字串格式
2. 將 parsed.search 更新至合併後的結果
透過 stringifyParsedURL(parsed) 得到合併後的字串結果

ufo

export function withQuery(input: string, query: QueryObject): string {
  const parsed = parseURL(input);
  const mergedQuery = { ...parseQuery(parsed.search), ...query };
  parsed.search = stringifyQuery(mergedQuery);
  return stringifyParsedURL(parsed);
}

`parseURL`

先簡單說明 parseURL 實作：透過正規表達式，拆解出 URI syntax 的各個部分

URI syntax

URI = scheme ":" ["//" authority] path ["?" query] ["#" fragment]
authority = [userinfo "@"] host [":" port]

URI example

        userinfo       host      port
        ┌──┴───┐ ┌──────┴──────┐ ┌┴─┐
https://john.doe@www.example.com:1234/forum/questions/?tag=networking&order=newest#top
└─┬─┘   └─────────────┬─────────────┘└───────┬───────┘ └────────────┬────────────┘ └┬┘
scheme            authority                path                   query          fragment

usage

parseURL(
  'https://john.doe@www.example.com:1234/forum/questions/?tag=networking&order=newest#top'
);
 
/**
 * {
 *   auth: "john.doe",
 *   hash: "#top",
 *   host: "www.example.com:1234",
 *   pathname: "/forum/questions/",
 *   protocol: "https:",
 *   search: "?tag=networking&amp;order=newest"
 * }
 */

實作細節：

多數完整路徑，透過 /^[\s\0]*([\w+.-]{2,}:)?\/\/([^/@]+@)?(.*)/ 分群

([\w+.-]{2,}:)?\/\/ 擷取 protocol（多數情況 protocol 後方會接著兩個斜線，例如：http://, ftp://）
([^/@]+@)? 擷取 userinfo（例如：user:pass@）
(.*) 剩餘的任意字元，對應到 URI 中的 host, port, path, query, fragment

function parseURL(input = "", defaultProto?: string): ParsedURL {
  // skip
  const [, protocol = "", auth, hostAndPath = ""] =
    input
      .replace(/\\/g, "/")
      .match(/^[\s\0]*([\w+.-]{2,}:)?\/\/([^/@]+@)?(.*)/) || [];
  /**
   * example:
   * https://john.doe@www.foo.com:1234/forum/questions/?order=newest#top
   * 
   * protocol: https:
   * auth: john.doe@
   * hostAndPath: www.foo.com:1234/forum/questions/?order=newest#top
   */
 
  // skip
}

考慮特殊的 protocol 格式，例如：

data:image/jpeg;base64,/9j/4AAQSk
blob:http://example.com/550e8400-e29b-41d4-a716-446655440000

function parseURL(input = "", defaultProto?: string): ParsedURL {
  const _specialProtoMatch = input.match(
    /^[\s\0]*(blob:|data:|javascript:|vbscript:)(.*)/i,
  );
  if (_specialProtoMatch) {
    const [, _proto, _pathname = ""] = _specialProtoMatch;
    return {
      protocol: _proto.toLowerCase(),
      pathname: _pathname,
      href: _proto + _pathname,
      auth: "",
      host: "",
      search: "",
      hash: "",
    };
  }
  // skip
  const [, protocol = "", auth, hostAndPath = ""] =
    input
      .replace(/\\/g, "/")
      .match(/^[\s\0]*([\w+.-]{2,}:)?\/\/([^/@]+@)?(.*)/) || [];
  // skip
}

支援 parse 相對路徑

function parseURL(input = "", defaultProto?: string): ParsedURL {
  const _specialProtoMatch = input.match(
    /^[\s\0]*(blob:|data:|javascript:|vbscript:)(.*)/i,
  );
  if (_specialProtoMatch) {
    const [, _proto, _pathname = ""] = _specialProtoMatch;
    return {
      protocol: _proto.toLowerCase(),
      pathname: _pathname,
      href: _proto + _pathname,
      auth: "",
      host: "",
      search: "",
      hash: "",
    };
  }
  if (!hasProtocol(input, { acceptRelative: true })) {
    return defaultProto ? parseURL(defaultProto + input) : parsePath(input);
  }
  const [, protocol = "", auth, hostAndPath = ""] =
    input
      .replace(/\\/g, "/")
      .match(/^[\s\0]*([\w+.-]{2,}:)?\/\/([^/@]+@)?(.*)/) || [];
  // skip
}

拆解出 host, path 後，解析 path

function parseURL(input = "", defaultProto?: string): ParsedURL {
  const _specialProtoMatch = input.match(
    /^[\s\0]*(blob:|data:|javascript:|vbscript:)(.*)/i,
  );
  if (_specialProtoMatch) {
    const [, _proto, _pathname = ""] = _specialProtoMatch;
    return {
      protocol: _proto.toLowerCase(),
      pathname: _pathname,
      href: _proto + _pathname,
      auth: "",
      host: "",
      search: "",
      hash: "",
    };
  }
 
  if (!hasProtocol(input, { acceptRelative: true })) {
    return defaultProto ? parseURL(defaultProto + input) : parsePath(input);
  }
 
  const [, protocol = "", auth, hostAndPath = ""] =
    input
      .replace(/\\/g, "/")
      .match(/^[\s\0]*([\w+.-]{2,}:)?\/\/([^/@]+@)?(.*)/) || [];
  const [, host = "", path = ""] = hostAndPath.match(/([^#/?]*)(.*)?/) || [];
  const { pathname, search, hash } = parsePath(
    path.replace(/\/(?=[A-Za-z]:)/, ""),
  );
 
  return {
    protocol: protocol.toLowerCase(),
    auth: auth ? auth.slice(0, Math.max(0, auth.length - 1)) : "",
    host,
    pathname,
    search,
    hash,
    [protocolRelative]: !protocol,
  };
}

`parsePath`

parsePath 的實作，也是透過正規表達式，將 path 拆解出相應的部分

parsePath('/foo?tag=bar#baz')
 
/**
 * {
 *   pathname: "/foo",
 *   search: "?tag=bar"
 *   hash: "#baz",
 * }
 */

/([^#?]*)(\?[^#]*)?(#.*)?/，以 /foo?query=bar#baz 為例：

([^#?]*)：任意非 # 或 ? 的字元序列，這裡擷取到 /foo
(\?[^#]*)?：零或一組以 ? 開頭，後面不包含 # 的字元序列，這裡擷取到 ?query=bar
(#.*)?：零或一組以 # 開頭的任意字元序列，這裡擷取到 #baz

function parsePath(input = "") {
  const [pathname = "", search = "", hash = ""] = (
    input.match(/([^#?]*)(\?[^#]*)?(#.*)?/) || []
  ).splice(1);
 
  return {
    pathname,
    search,
    hash,
  };
}

`parseQuery`

parseQuery('?test=123&foo=456&bar=789');
 
/**
 * {
 *   bar: "789",
 *   foo: "456",
 *   test: "123"
 *  }
 */

如果 input 開頭包含 ?，則透過 slice(1) 清除

function parseQuery<T extends ParsedQuery = ParsedQuery>(
  parametersString = "",
): T {
  const object: ParsedQuery = {};
  if (parametersString[0] === "?") {
    parametersString = parametersString.slice(1);
  }
  // skip
  return object as T;
}

透過 split('&') 轉換成陣列，執行 for of loop

function parseQuery<T extends ParsedQuery = ParsedQuery>(
  parametersString = "",
): T {
  const object: ParsedQuery = {};
  if (parametersString[0] === "?") {
    parametersString = parametersString.slice(1);
  }
  for (const parameter of parametersString.split("&")) {
    // skip
  }
  return object as T;
}

透過 /([^=]+)=?(.*)/ 拆解出 key, value，以 test=123 為例：

([^=]+) 擷取到 test
?(.*) 擷取到 123

function parseQuery<T extends ParsedQuery = ParsedQuery>(
  parametersString = "",
): T {
  const object: ParsedQuery = {};
  if (parametersString[0] === "?") {
    parametersString = parametersString.slice(1);
  }
  for (const parameter of parametersString.split("&")) {
    const s = parameter.match(/([^=]+)=?(.*)/) || [];
    if (s.length < 2) {
      continue;
    }
    const key = decodeQueryKey(s[1]);
    const value = decodeQueryValue(s[2] || "");
    // skip
  }
  return object as T;
}

避免 prototype pollution，略過 key 為 __proto__ 或 constructor 的 parameter

function parseQuery<T extends ParsedQuery = ParsedQuery>(
  parametersString = "",
): T {
  const object: ParsedQuery = {};
  if (parametersString[0] === "?") {
    parametersString = parametersString.slice(1);
  }
  for (const parameter of parametersString.split("&")) {
    const s = parameter.match(/([^=]+)=?(.*)/) || [];
    if (s.length < 2) {
      continue;
    }
    const key = decodeQueryKey(s[1]);
    if (key === "__proto__" || key === "constructor") {
      continue;
    }
    const value = decodeQueryValue(s[2] || "");
    // skip
  }
  return object as T;
}

考慮 multi-value 的情境

function parseQuery<T extends ParsedQuery = ParsedQuery>(
  parametersString = "",
): T {
  const object: ParsedQuery = {};
  if (parametersString[0] === "?") {
    parametersString = parametersString.slice(1);
  }
  for (const parameter of parametersString.split("&")) {
    const s = parameter.match(/([^=]+)=?(.*)/) || [];
    if (s.length < 2) {
      continue;
    }
    const key = decodeQueryKey(s[1]);
    const value = decodeQueryValue(s[2] || "");
    if (object[key] === undefined) {
      object[key] = value;
    } else if (Array.isArray(object[key])) {
      (object[key] as string[]).push(value);
    } else {
      object[key] = [object[key] as string, value];
    }
  }
  return object as T;
}

下方附上 decodeQueryKey, decodeQueryValue 的程式碼

const PLUS_RE = /\+/g; // %2B
 
function decodeQueryKey(text: string): string {
  return decode(text.replace(PLUS_RE, " "));
}
 
function decodeQueryValue(text: string): string {
  return decode(text.replace(PLUS_RE, " "));
}
 
/**
 * Decode text using `decodeURIComponent`. Returns the original text if it
 * fails.
 */
function decode(text: string | number = ""): string {
  try {
    return decodeURIComponent("" + text);
  } catch {
    return "" + text;
  }
}

`stringifyQuery`

stringifyQuery({
  foo: [123, 456],
  bar: true,
  hello: 'world'
});
 
// foo=123&foo=456&bar=true&hello=world

stringifyQuery 會調用 encodeQueryItem key, value 轉換成字串

function stringifyQuery(query: QueryObject): string {
  return Object.keys(query)
    .filter((k) => query[k] !== undefined)
    .map((k) => encodeQueryItem(k, query[k]))
    .filter(Boolean)
    .join("&");
}

encodeQueryItem 的處理方式：

如果 typeof value 為 number 或 boolean，會被轉換成字串，例如： ?foo=123&bar=true
如果 value 為 array，轉換成多個 key-value pairs，例如： ?foo=123&foo=456

function encodeQueryItem(
  key: string,
  value: QueryValue | QueryValue[],
): string {
  if (typeof value === "number" || typeof value === "boolean") {
    value = String(value);
  }
  if (!value) {
    return encodeQueryKey(key);
  }
  if (Array.isArray(value)) {
    return value
      .map((_value) => `${encodeQueryKey(key)}=${encodeQueryValue(_value)}`)
      .join("&");
  }
 
  return `${encodeQueryKey(key)}=${encodeQueryValue(value)}`;
}

`stringifyParsedURL`

按照 URI syntax 將各個部分組合成字串格式

function stringifyParsedURL(parsed: Partial<ParsedURL>): string {
  const pathname = parsed.pathname || "";
  const search = parsed.search
    ? (parsed.search.startsWith("?") ? "" : "?") + parsed.search
    : "";
  const hash = parsed.hash || "";
  const auth = parsed.auth ? parsed.auth + "@" : "";
  const host = parsed.host || "";
  const proto =
    parsed.protocol || parsed[protocolRelative]
      ? (parsed.protocol || "") + "//"
      : "";
  return proto + auth + host + pathname + search + hash;
}

Inside unjs/ofetch query search params

parseURL

parsePath

parseQuery

stringifyQuery

stringifyParsedURL

Reference

`parseURL`

`parsePath`

`parseQuery`

`stringifyQuery`

`stringifyParsedURL`