This commit is contained in:
Your Name
2026-07-23 17:56:25 +08:00
parent a05dae8412
commit 4970d8f8d3
4262 changed files with 735221 additions and 0 deletions
Generated Vendored
+13
View File
@@ -0,0 +1,13 @@
Copyright (c) 2017 Thomas Parisot, 2018 Rémi Berson
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+368
View File
@@ -0,0 +1,368 @@
# tldts - Blazing Fast URL Parsing
`tldts` is a JavaScript library to extract hostnames, domains, public suffixes, top-level domains and subdomains from URLs.
**Features**:
1. Tuned for **performance** (order of 0.1 to 1 μs per input)
2. Handles both URLs and hostnames
3. Full Unicode/IDNA support
4. Support parsing email addresses
5. Detect IPv4 and IPv6 addresses
6. Continuously updated version of the public suffix list
7. **TypeScript**, ships with `umd`, `esm`, `cjs` bundles and _type definitions_
8. Small bundles and small memory footprint
9. Battle tested: full test coverage and production use
# Install
```bash
npm install --save tldts
```
# Usage
Using the command-line interface:
```js
$ npx tldts 'http://www.writethedocs.org/conf/eu/2017/'
{
"domain": "writethedocs.org",
"domainWithoutSuffix": "writethedocs",
"hostname": "www.writethedocs.org",
"isIcann": true,
"isIp": false,
"isPrivate": false,
"publicSuffix": "org",
"subdomain": "www"
}
```
Programmatically:
```js
const { parse } = require('tldts');
// Retrieving hostname related informations of a given URL
parse('http://www.writethedocs.org/conf/eu/2017/');
// { domain: 'writethedocs.org',
// domainWithoutSuffix: 'writethedocs',
// hostname: 'www.writethedocs.org',
// isIcann: true,
// isIp: false,
// isPrivate: false,
// publicSuffix: 'org',
// subdomain: 'www' }
```
Modern _ES6 modules import_ is also supported:
```js
import { parse } from 'tldts';
```
Alternatively, you can try it _directly in your browser_ here: https://npm.runkit.com/tldts
# API
- `tldts.parse(url | hostname, options)`
- `tldts.getHostname(url | hostname, options)`
- `tldts.getDomain(url | hostname, options)`
- `tldts.getFullDomain(url | hostname, options)`
- `tldts.getPublicSuffix(url | hostname, options)`
- `tldts.getSubdomain(url, | hostname, options)`
- `tldts.getDomainWithoutSuffix(url | hostname, options)`
The behavior of `tldts` can be customized using an `options` argument for all
the functions exposed as part of the public API. This is useful to both change
the behavior of the library as well as fine-tune the performance depending on
your inputs.
```js
{
// Use suffixes from ICANN section (default: true)
allowIcannDomains: boolean;
// Use suffixes from Private section (default: false)
allowPrivateDomains: boolean;
// Extract and validate hostname (default: true)
// When set to `false`, inputs will be considered valid hostnames.
extractHostname: boolean;
// Validate hostnames after parsing (default: true)
// If a hostname is not valid, not further processing is performed. When set
// to `false`, inputs to the library will be considered valid and parsing will
// proceed regardless.
validateHostname: boolean;
// Perform IP address detection (default: true).
detectIp: boolean;
// Detect IANA special-use domains (RFC 6761 et al.) and expose the result as
// `isSpecialUse` (default: false). Off by default so the common path does no
// extra work; the field stays `null` unless this is enabled.
detectSpecialUse: boolean;
// Assume that both URLs and hostnames can be given as input (default: true)
// If set to `false` we assume only URLs will be given as input, which
// speed-ups processing.
mixedInputs: boolean;
// Specifies extra valid suffixes (default: null)
validHosts: string[] | null;
}
```
The `parse` method returns handy **properties about a URL or a hostname**.
```js
const tldts = require('tldts');
tldts.parse('https://spark-public.s3.amazonaws.com/dataanalysis/loansData.csv');
// { domain: 'amazonaws.com',
// domainWithoutSuffix: 'amazonaws',
// hostname: 'spark-public.s3.amazonaws.com',
// isIcann: true,
// isIp: false,
// isPrivate: false,
// publicSuffix: 'com',
// subdomain: 'spark-public.s3' }
tldts.parse(
'https://spark-public.s3.amazonaws.com/dataanalysis/loansData.csv',
{ allowPrivateDomains: true },
);
// { domain: 'spark-public.s3.amazonaws.com',
// domainWithoutSuffix: 'spark-public',
// hostname: 'spark-public.s3.amazonaws.com',
// isIcann: false,
// isIp: false,
// isPrivate: true,
// publicSuffix: 's3.amazonaws.com',
// subdomain: '' }
tldts.parse('gopher://domain.unknown/');
// { domain: 'domain.unknown',
// domainWithoutSuffix: 'domain',
// hostname: 'domain.unknown',
// isIcann: false,
// isIp: false,
// isPrivate: false,
// publicSuffix: 'unknown',
// subdomain: '' }
tldts.parse('https://192.168.0.0'); // IPv4
// { domain: null,
// domainWithoutSuffix: null,
// hostname: '192.168.0.0',
// isIcann: null,
// isIp: true,
// isPrivate: null,
// publicSuffix: null,
// subdomain: null }
tldts.parse('https://[::1]'); // IPv6
// { domain: null,
// domainWithoutSuffix: null,
// hostname: '::1',
// isIcann: null,
// isIp: true,
// isPrivate: null,
// publicSuffix: null,
// subdomain: null }
tldts.parse('tldts@emailprovider.co.uk'); // email
// { domain: 'emailprovider.co.uk',
// domainWithoutSuffix: 'emailprovider',
// hostname: 'emailprovider.co.uk',
// isIcann: true,
// isIp: false,
// isPrivate: false,
// publicSuffix: 'co.uk',
// subdomain: '' }
```
| Property Name | Type | Description |
| :-------------------- | :----- | :---------------------------------------------- |
| `hostname` | `str` | `hostname` of the input extracted automatically |
| `domain` | `str` | Domain (tld + sld) |
| `domainWithoutSuffix` | `str` | Domain without public suffix |
| `subdomain` | `str` | Sub domain (what comes after `domain`) |
| `publicSuffix` | `str` | Public Suffix (tld) of `hostname` |
| `isIcann` | `bool` | Does TLD come from ICANN part of the list |
| `isPrivate` | `bool` | Does TLD come from Private part of the list |
| `isIP` | `bool` | Is `hostname` an IP address? |
| `isSpecialUse` | `bool` | Is `hostname` an IANA special-use domain? |
## Special-use domains (RFC 6761 / IANA)
Set `{ detectSpecialUse: true }` to flag reserved special-use names such as `localhost`, `*.test`, `*.local`, `*.onion`, and `home.arpa` via the `isSpecialUse` result field. `isIcann`/`isPrivate` don't identify these: most aren't in the Public Suffix List, and the few that are (e.g. `onion`, `home.arpa`) appear there as ordinary ICANN suffixes. The field is `null` unless the option is enabled, so the default path does no extra work:
```js
parse('http://printer.local/', { detectSpecialUse: true });
// { ...
// isSpecialUse: true,
// publicSuffix: 'local',
// subdomain: '' }
```
The list tracks the IANA [Special-Use Domain Names](https://www.iana.org/assignments/special-use-domain-names/) registry.
## Single purpose methods
These methods are shorthands if you want to retrieve only a single value (and
will perform better than `parse` because less work will be needed).
### getHostname(url | hostname, options?)
Returns the hostname from a given string.
```javascript
const { getHostname } = require('tldts');
getHostname('google.com'); // returns `google.com`
getHostname('fr.google.com'); // returns `fr.google.com`
getHostname('fr.google.google'); // returns `fr.google.google`
getHostname('foo.google.co.uk'); // returns `foo.google.co.uk`
getHostname('t.co'); // returns `t.co`
getHostname('fr.t.co'); // returns `fr.t.co`
getHostname(
'https://user:password@example.co.uk:8080/some/path?and&query#hash',
); // returns `example.co.uk`
```
### getDomain(url | hostname, options?)
Returns the fully qualified domain from a given string.
```javascript
const { getDomain } = require('tldts');
getDomain('google.com'); // returns `google.com`
getDomain('fr.google.com'); // returns `google.com`
getDomain('fr.google.google'); // returns `google.google`
getDomain('foo.google.co.uk'); // returns `google.co.uk`
getDomain('t.co'); // returns `t.co`
getDomain('fr.t.co'); // returns `t.co`
getDomain('https://user:password@example.co.uk:8080/some/path?and&query#hash'); // returns `example.co.uk`
```
### getFullDomain(url | hostname, options?)
Returns the full domain — the subdomain together with the registrable domain (as
returned by `getDomain(...)`), i.e. the whole hostname _including_ any subdomain —
or `null` when the input has no registrable domain (IP address, single label,
bare public suffix, …). The result is the normalized hostname (lower-cased,
trailing dot stripped); it is not a DNS-absolute name (no trailing root dot) and
no IDNA/punycode conversion is performed.
```javascript
const { getFullDomain } = require('tldts');
getFullDomain('google.com'); // returns `google.com`
getFullDomain('fr.google.com'); // returns `fr.google.com`
getFullDomain('foo.google.co.uk'); // returns `foo.google.co.uk`
getFullDomain('t.co'); // returns `t.co`
getFullDomain('fr.t.co'); // returns `fr.t.co`
getFullDomain('1.2.3.4'); // returns null (no registrable domain)
getFullDomain('localhost'); // returns null
```
### getDomainWithoutSuffix(url | hostname, options?)
Returns the domain (as returned by `getDomain(...)`) without the public suffix part.
```javascript
const { getDomainWithoutSuffix } = require('tldts');
getDomainWithoutSuffix('google.com'); // returns `google`
getDomainWithoutSuffix('fr.google.com'); // returns `google`
getDomainWithoutSuffix('fr.google.google'); // returns `google`
getDomainWithoutSuffix('foo.google.co.uk'); // returns `google`
getDomainWithoutSuffix('t.co'); // returns `t`
getDomainWithoutSuffix('fr.t.co'); // returns `t`
getDomainWithoutSuffix(
'https://user:password@example.co.uk:8080/some/path?and&query#hash',
); // returns `example`
```
### getSubdomain(url | hostname, options?)
Returns the complete subdomain for a given string.
```javascript
const { getSubdomain } = require('tldts');
getSubdomain('google.com'); // returns ``
getSubdomain('fr.google.com'); // returns `fr`
getSubdomain('google.co.uk'); // returns ``
getSubdomain('foo.google.co.uk'); // returns `foo`
getSubdomain('moar.foo.google.co.uk'); // returns `moar.foo`
getSubdomain('t.co'); // returns ``
getSubdomain('fr.t.co'); // returns `fr`
getSubdomain(
'https://user:password@secure.example.co.uk:443/some/path?and&query#hash',
); // returns `secure`
```
### getPublicSuffix(url | hostname, options?)
Returns the [public suffix][] for a given string.
```javascript
const { getPublicSuffix } = require('tldts');
getPublicSuffix('google.com'); // returns `com`
getPublicSuffix('fr.google.com'); // returns `com`
getPublicSuffix('google.co.uk'); // returns `co.uk`
getPublicSuffix('s3.amazonaws.com'); // returns `com`
getPublicSuffix('s3.amazonaws.com', { allowPrivateDomains: true }); // returns `s3.amazonaws.com`
getPublicSuffix('tld.is.unknown'); // returns `unknown`
```
# Troubleshooting
## Retrieving subdomain of `localhost` and custom hostnames
`tldts` methods `getDomain` and `getSubdomain` are designed to **work only with _known and valid_ TLDs**.
This way, you can trust what a domain is.
`localhost` is a valid hostname but not a TLD. You can pass additional options to each method exposed by `tldts`:
```js
const tldts = require('tldts');
tldts.getDomain('localhost'); // returns null
tldts.getSubdomain('vhost.localhost'); // returns null
tldts.getDomain('localhost', { validHosts: ['localhost'] }); // returns 'localhost'
tldts.getSubdomain('vhost.localhost', { validHosts: ['localhost'] }); // returns 'vhost'
```
## Updating the TLDs List
`tldts` made the opinionated choice of shipping with a list of suffixes directly
in its bundle. There is currently no mechanism to update the lists yourself, but
we make sure that the version shipped is always up-to-date.
If you keep `tldts` updated, the lists should be up-to-date as well!
# Performance
`tldts` is the _fastest JavaScript library_ available for parsing hostnames. It is able to parse _millions of inputs per second_ (typically 2-3M depending on your hardware and inputs). It also offers granular options to fine-tune the behavior and performance of the library depending on the kind of inputs you are dealing with (e.g.: if you know you only manipulate valid hostnames you can disable the hostname extraction step with `{ extractHostname: false }`).
Please see [this detailed comparison](./comparison/comparison.md) with other available libraries.
## Contributors
`tldts` is based upon the excellent `tld.js` library and would not exist without
the many contributors who worked on the project:
<a href="graphs/contributors"><img src="https://opencollective.com/tldjs/contributors.svg?width=890" /></a>
This project would not be possible without the amazing Mozilla's
[public suffix list][]. Thank you for your hard work!
# License
[MIT License](LICENSE).
[badge-ci]: https://secure.travis-ci.org/remusao/tldts.svg?branch=master
[badge-downloads]: https://img.shields.io/npm/dm/tldts.svg
[public suffix list]: https://publicsuffix.org/list/
[list the recent changes]: https://github.com/publicsuffix/list/commits/master
[changes Atom Feed]: https://github.com/publicsuffix/list/commits/master.atom
[public suffix]: https://publicsuffix.org/learn/
+21
View File
@@ -0,0 +1,21 @@
#!/usr/bin/env node
'use strict';
const { parse } = require('..');
const readline = require('readline');
if (process.argv.length > 2) {
// URL(s) was specified in the command arguments
console.log(
JSON.stringify(parse(process.argv[process.argv.length - 1]), null, 2),
);
} else {
// No arguments were specified, read URLs from each line of STDIN
const rlInterface = readline.createInterface({
input: process.stdin,
});
rlInterface.on('line', function (line) {
console.log(JSON.stringify(parse(line), null, 2));
});
}
+1170
View File
File diff suppressed because one or more lines are too long
+1
View File
File diff suppressed because one or more lines are too long
+12
View File
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+170
View File
@@ -0,0 +1,170 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.default = suffixLookup;
// NOTE: kept (intentionally) near-identical to packages/tldts-icann/src/suffix-trie.ts.
// They are separate copies rather than a shared helper because the lookup is
// only fast when the typed arrays are module-scope monomorphic globals —
// closing over them (a shared factory) measured ~20% slower. The ICANN build
// also specializes (constant mask, no isIcann/isPrivate). Keep the two in sync.
const tldts_core_1 = require("tldts-core");
const trie_1 = require("./data/trie");
// `edgeOffset` (where each label starts in `labelText`), `edgeHash` (djb2 of
// each label) and `wildcardEdge` (each node's '*' edge, or -1) are derived once
// at load instead of being shipped: the bundle then carries only the
// compressible `labelText` + structure, while the lookup binary-searches
// integer hashes. The cost is a single ~1ms pass at first import — cheaper than
// the object trie it replaces. Kept at module scope (not captured in a closure)
// so V8 treats the typed arrays as fast monomorphic globals.
const numberOfNodes = trie_1.nodeFlags.length;
const numberOfEdges = trie_1.edgeLength.length;
const edgeOffset = new Uint32Array(numberOfEdges);
const edgeHash = new Uint32Array(numberOfEdges);
const wildcardEdge = new Int32Array(numberOfNodes).fill(-1);
for (let node = 0, offset = 0; node < numberOfNodes; node += 1) {
for (let edge = trie_1.edgeStart[node]; edge < trie_1.edgeStart[node + 1]; edge += 1) {
edgeOffset[edge] = offset;
const end = offset + trie_1.edgeLength[edge];
let hash = 5381;
for (let i = end - 1; i >= offset; i -= 1) {
hash = (hash * 33) ^ trie_1.labelText.charCodeAt(i);
}
edgeHash[edge] = hash >>> 0;
if (trie_1.edgeLength[edge] === 1 &&
trie_1.labelText.charCodeAt(offset) === 42 /* '*' */) {
wildcardEdge[node] = edge;
}
offset = end;
}
}
// Result of the last `walk`, kept in module scope to avoid allocating a match
// object. Safe because lookups are synchronous and read right after `walk`.
let matchNode = -1;
let matchStart = 0;
let matchEnd = 0;
/**
* True if edge `edge`'s label equals `hostname[start, start + length)`.
*/
function labelEquals(edge, hostname, start, length) {
if (trie_1.edgeLength[edge] !== length) {
return false;
}
const offset = edgeOffset[edge];
for (let i = 0; i < length; i += 1) {
if (trie_1.labelText.charCodeAt(offset + i) !== hostname.charCodeAt(start + i)) {
return false;
}
}
return true;
}
/**
* Find the child edge of `node` whose label is `hostname[start, start + length)`.
* Edges are sorted by hash, so binary-search the hash then verify the label
* (scanning the rare run of equal hashes). Returns the edge index or -1.
*/
function findEdge(node, hash, hostname, start, length) {
let lo = trie_1.edgeStart[node];
let hi = trie_1.edgeStart[node + 1];
while (lo < hi) {
const mid = (lo + hi) >>> 1;
const value = edgeHash[mid];
if (value < hash) {
lo = mid + 1;
}
else if (value > hash) {
hi = mid;
}
else {
for (let e = mid; e >= lo && edgeHash[e] === hash; e -= 1) {
if (labelEquals(e, hostname, start, length))
return e;
}
for (let e = mid + 1; e < hi && edgeHash[e] === hash; e += 1) {
if (labelEquals(e, hostname, start, length))
return e;
}
return -1;
}
}
return -1;
}
/**
* Walk `hostname`'s labels right-to-left from `root`, recording the deepest
* node whose flag passes `allowedMask` (with the label boundaries of that match
* in `matchStart`/`matchEnd`). Returns whether any match was found.
*/
function walk(hostname, root, allowedMask) {
let node = root;
let end = hostname.length;
let hash = 5381;
matchNode = -1;
for (let i = hostname.length - 1; i >= 0; i -= 1) {
const code = hostname.charCodeAt(i);
if (code === 46 /* '.' */) {
const start = i + 1;
let edge = findEdge(node, hash >>> 0, hostname, start, end - start);
if (edge === -1) {
edge = wildcardEdge[node];
}
if (edge === -1) {
return matchNode !== -1;
}
node = trie_1.edgeChild[edge];
if ((trie_1.nodeFlags[node] & allowedMask) !== 0) {
matchNode = node;
matchStart = start;
matchEnd = end;
}
end = i;
hash = 5381;
}
else {
hash = (hash * 33) ^ code;
}
}
// Left-most label: hostname[0, end). Same find/descend/record as the loop —
// duplicated rather than folded into the loop (via `i >= -1`) because that
// extra per-character branch measured slightly slower on the hot path.
let edge = findEdge(node, hash >>> 0, hostname, 0, end);
if (edge === -1) {
edge = wildcardEdge[node];
}
if (edge !== -1) {
node = trie_1.edgeChild[edge];
if ((trie_1.nodeFlags[node] & allowedMask) !== 0) {
matchNode = node;
matchStart = 0;
matchEnd = end;
}
}
return matchNode !== -1;
}
/**
* Check if `hostname` has a valid public suffix in the trie.
*/
function suffixLookup(hostname, options, out) {
if ((0, tldts_core_1.fastPathLookup)(hostname, options, out)) {
return;
}
const allowedMask = (options.allowPrivateDomains ? 2 /* RULE_TYPE.PRIVATE */ : 0) |
(options.allowIcannDomains ? 1 /* RULE_TYPE.ICANN */ : 0);
// Exceptions have priority and strip their own left-most label (e.g. the
// rule '!www.ck' makes the suffix of 'www.ck' be 'ck').
if (walk(hostname, trie_1.exceptionsRoot, allowedMask)) {
out.isIcann = (trie_1.nodeFlags[matchNode] & 1 /* RULE_TYPE.ICANN */) !== 0;
out.isPrivate = (trie_1.nodeFlags[matchNode] & 2 /* RULE_TYPE.PRIVATE */) !== 0;
out.publicSuffix = hostname.slice(matchEnd + 1);
return;
}
if (walk(hostname, trie_1.rulesRoot, allowedMask)) {
out.isIcann = (trie_1.nodeFlags[matchNode] & 1 /* RULE_TYPE.ICANN */) !== 0;
out.isPrivate = (trie_1.nodeFlags[matchNode] & 2 /* RULE_TYPE.PRIVATE */) !== 0;
out.publicSuffix = hostname.slice(matchStart);
return;
}
// No match: the prevailing '*' rule makes the right-most label the suffix.
out.isIcann = false;
out.isPrivate = false;
const lastDot = hostname.lastIndexOf('.');
out.publicSuffix = lastDot === -1 ? hostname : hostname.slice(lastDot + 1);
}
//# sourceMappingURL=suffix-trie.js.map
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+41
View File
@@ -0,0 +1,41 @@
import { getEmptyResult, parseImpl, resetResult, } from 'tldts-core';
import suffixLookup from './src/suffix-trie';
// For all methods but 'parse', it does not make sense to allocate an object
// every single time to only return the value of a specific attribute. To avoid
// this un-necessary allocation, we use a global object which is re-used.
const RESULT = getEmptyResult();
export function parse(url, options) {
return parseImpl(url, 5 /* FLAG.ALL */, suffixLookup, options, getEmptyResult());
}
export function getHostname(url, options) {
/*@__INLINE__*/ resetResult(RESULT);
return parseImpl(url, 0 /* FLAG.HOSTNAME */, suffixLookup, options, RESULT).hostname;
}
export function getPublicSuffix(url, options) {
/*@__INLINE__*/ resetResult(RESULT);
return parseImpl(url, 2 /* FLAG.PUBLIC_SUFFIX */, suffixLookup, options, RESULT)
.publicSuffix;
}
export function getDomain(url, options) {
/*@__INLINE__*/ resetResult(RESULT);
return parseImpl(url, 3 /* FLAG.DOMAIN */, suffixLookup, options, RESULT).domain;
}
export function getFullDomain(url, options) {
/*@__INLINE__*/ resetResult(RESULT);
const result = parseImpl(url, 3 /* FLAG.DOMAIN */, suffixLookup, options, RESULT);
// The hostname *is* the full domain (subdomain + domain) whenever a
// registrable domain exists; gate on `domain` so non-registrable inputs
// (IPs, suffix-less or invalid hostnames) return `null` like `getDomain`.
return result.domain === null ? null : result.hostname;
}
export function getSubdomain(url, options) {
/*@__INLINE__*/ resetResult(RESULT);
return parseImpl(url, 4 /* FLAG.SUB_DOMAIN */, suffixLookup, options, RESULT)
.subdomain;
}
export function getDomainWithoutSuffix(url, options) {
/*@__INLINE__*/ resetResult(RESULT);
return parseImpl(url, 5 /* FLAG.ALL */, suffixLookup, options, RESULT)
.domainWithoutSuffix;
}
//# sourceMappingURL=index.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../index.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,cAAc,EAGd,SAAS,EACT,WAAW,GACZ,MAAM,YAAY,CAAC;AAEpB,OAAO,YAAY,MAAM,mBAAmB,CAAC;AAE7C,4EAA4E;AAC5E,+EAA+E;AAC/E,yEAAyE;AACzE,MAAM,MAAM,GAAY,cAAc,EAAE,CAAC;AAEzC,MAAM,UAAU,KAAK,CAAC,GAAW,EAAE,OAA2B;IAC5D,OAAO,SAAS,CAAC,GAAG,oBAAY,YAAY,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,GAAW,EACX,OAA2B;IAE3B,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,SAAS,CAAC,GAAG,yBAAiB,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC;AAC/E,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,GAAW,EACX,OAA2B;IAE3B,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,SAAS,CAAC,GAAG,8BAAsB,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC;SACrE,YAAY,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,GAAW,EACX,OAA2B;IAE3B,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,SAAS,CAAC,GAAG,uBAAe,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC;AAC3E,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,GAAW,EACX,OAA2B;IAE3B,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,uBAAe,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC1E,oEAAoE;IACpE,wEAAwE;IACxE,0EAA0E;IAC1E,OAAO,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,GAAW,EACX,OAA2B;IAE3B,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,SAAS,CAAC,GAAG,2BAAmB,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC;SAClE,SAAS,CAAC;AACf,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,GAAW,EACX,OAA2B;IAE3B,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,SAAS,CAAC,GAAG,oBAAY,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC;SAC3D,mBAAmB,CAAC;AACzB,CAAC"}
+9
View File
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+167
View File
@@ -0,0 +1,167 @@
// NOTE: kept (intentionally) near-identical to packages/tldts-icann/src/suffix-trie.ts.
// They are separate copies rather than a shared helper because the lookup is
// only fast when the typed arrays are module-scope monomorphic globals —
// closing over them (a shared factory) measured ~20% slower. The ICANN build
// also specializes (constant mask, no isIcann/isPrivate). Keep the two in sync.
import { fastPathLookup, } from 'tldts-core';
import { edgeChild, edgeLength, edgeStart, exceptionsRoot, labelText, nodeFlags, rulesRoot, } from './data/trie';
// `edgeOffset` (where each label starts in `labelText`), `edgeHash` (djb2 of
// each label) and `wildcardEdge` (each node's '*' edge, or -1) are derived once
// at load instead of being shipped: the bundle then carries only the
// compressible `labelText` + structure, while the lookup binary-searches
// integer hashes. The cost is a single ~1ms pass at first import — cheaper than
// the object trie it replaces. Kept at module scope (not captured in a closure)
// so V8 treats the typed arrays as fast monomorphic globals.
const numberOfNodes = nodeFlags.length;
const numberOfEdges = edgeLength.length;
const edgeOffset = new Uint32Array(numberOfEdges);
const edgeHash = new Uint32Array(numberOfEdges);
const wildcardEdge = new Int32Array(numberOfNodes).fill(-1);
for (let node = 0, offset = 0; node < numberOfNodes; node += 1) {
for (let edge = edgeStart[node]; edge < edgeStart[node + 1]; edge += 1) {
edgeOffset[edge] = offset;
const end = offset + edgeLength[edge];
let hash = 5381;
for (let i = end - 1; i >= offset; i -= 1) {
hash = (hash * 33) ^ labelText.charCodeAt(i);
}
edgeHash[edge] = hash >>> 0;
if (edgeLength[edge] === 1 &&
labelText.charCodeAt(offset) === 42 /* '*' */) {
wildcardEdge[node] = edge;
}
offset = end;
}
}
// Result of the last `walk`, kept in module scope to avoid allocating a match
// object. Safe because lookups are synchronous and read right after `walk`.
let matchNode = -1;
let matchStart = 0;
let matchEnd = 0;
/**
* True if edge `edge`'s label equals `hostname[start, start + length)`.
*/
function labelEquals(edge, hostname, start, length) {
if (edgeLength[edge] !== length) {
return false;
}
const offset = edgeOffset[edge];
for (let i = 0; i < length; i += 1) {
if (labelText.charCodeAt(offset + i) !== hostname.charCodeAt(start + i)) {
return false;
}
}
return true;
}
/**
* Find the child edge of `node` whose label is `hostname[start, start + length)`.
* Edges are sorted by hash, so binary-search the hash then verify the label
* (scanning the rare run of equal hashes). Returns the edge index or -1.
*/
function findEdge(node, hash, hostname, start, length) {
let lo = edgeStart[node];
let hi = edgeStart[node + 1];
while (lo < hi) {
const mid = (lo + hi) >>> 1;
const value = edgeHash[mid];
if (value < hash) {
lo = mid + 1;
}
else if (value > hash) {
hi = mid;
}
else {
for (let e = mid; e >= lo && edgeHash[e] === hash; e -= 1) {
if (labelEquals(e, hostname, start, length))
return e;
}
for (let e = mid + 1; e < hi && edgeHash[e] === hash; e += 1) {
if (labelEquals(e, hostname, start, length))
return e;
}
return -1;
}
}
return -1;
}
/**
* Walk `hostname`'s labels right-to-left from `root`, recording the deepest
* node whose flag passes `allowedMask` (with the label boundaries of that match
* in `matchStart`/`matchEnd`). Returns whether any match was found.
*/
function walk(hostname, root, allowedMask) {
let node = root;
let end = hostname.length;
let hash = 5381;
matchNode = -1;
for (let i = hostname.length - 1; i >= 0; i -= 1) {
const code = hostname.charCodeAt(i);
if (code === 46 /* '.' */) {
const start = i + 1;
let edge = findEdge(node, hash >>> 0, hostname, start, end - start);
if (edge === -1) {
edge = wildcardEdge[node];
}
if (edge === -1) {
return matchNode !== -1;
}
node = edgeChild[edge];
if ((nodeFlags[node] & allowedMask) !== 0) {
matchNode = node;
matchStart = start;
matchEnd = end;
}
end = i;
hash = 5381;
}
else {
hash = (hash * 33) ^ code;
}
}
// Left-most label: hostname[0, end). Same find/descend/record as the loop —
// duplicated rather than folded into the loop (via `i >= -1`) because that
// extra per-character branch measured slightly slower on the hot path.
let edge = findEdge(node, hash >>> 0, hostname, 0, end);
if (edge === -1) {
edge = wildcardEdge[node];
}
if (edge !== -1) {
node = edgeChild[edge];
if ((nodeFlags[node] & allowedMask) !== 0) {
matchNode = node;
matchStart = 0;
matchEnd = end;
}
}
return matchNode !== -1;
}
/**
* Check if `hostname` has a valid public suffix in the trie.
*/
export default function suffixLookup(hostname, options, out) {
if (fastPathLookup(hostname, options, out)) {
return;
}
const allowedMask = (options.allowPrivateDomains ? 2 /* RULE_TYPE.PRIVATE */ : 0) |
(options.allowIcannDomains ? 1 /* RULE_TYPE.ICANN */ : 0);
// Exceptions have priority and strip their own left-most label (e.g. the
// rule '!www.ck' makes the suffix of 'www.ck' be 'ck').
if (walk(hostname, exceptionsRoot, allowedMask)) {
out.isIcann = (nodeFlags[matchNode] & 1 /* RULE_TYPE.ICANN */) !== 0;
out.isPrivate = (nodeFlags[matchNode] & 2 /* RULE_TYPE.PRIVATE */) !== 0;
out.publicSuffix = hostname.slice(matchEnd + 1);
return;
}
if (walk(hostname, rulesRoot, allowedMask)) {
out.isIcann = (nodeFlags[matchNode] & 1 /* RULE_TYPE.ICANN */) !== 0;
out.isPrivate = (nodeFlags[matchNode] & 2 /* RULE_TYPE.PRIVATE */) !== 0;
out.publicSuffix = hostname.slice(matchStart);
return;
}
// No match: the prevailing '*' rule makes the right-most label the suffix.
out.isIcann = false;
out.isPrivate = false;
const lastDot = hostname.lastIndexOf('.');
out.publicSuffix = lastDot === -1 ? hostname : hostname.slice(lastDot + 1);
}
//# sourceMappingURL=suffix-trie.js.map
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+2
View File
File diff suppressed because one or more lines are too long
+1
View File
File diff suppressed because one or more lines are too long
+2
View File
File diff suppressed because one or more lines are too long
+1
View File
File diff suppressed because one or more lines are too long
+2
View File
File diff suppressed because one or more lines are too long
+1
View File
File diff suppressed because one or more lines are too long
+8
View File
@@ -0,0 +1,8 @@
import { IOptions, IResult } from 'tldts-core';
export declare function parse(url: string, options?: Partial<IOptions>): IResult;
export declare function getHostname(url: string, options?: Partial<IOptions>): string | null;
export declare function getPublicSuffix(url: string, options?: Partial<IOptions>): string | null;
export declare function getDomain(url: string, options?: Partial<IOptions>): string | null;
export declare function getFullDomain(url: string, options?: Partial<IOptions>): string | null;
export declare function getSubdomain(url: string, options?: Partial<IOptions>): string | null;
export declare function getDomainWithoutSuffix(url: string, options?: Partial<IOptions>): string | null;
File diff suppressed because one or more lines are too long
+5
View File
@@ -0,0 +1,5 @@
import { IPublicSuffix, ISuffixLookupOptions } from 'tldts-core';
/**
* Check if `hostname` has a valid public suffix in the trie.
*/
export default function suffixLookup(hostname: string, options: ISuffixLookupOptions, out: IPublicSuffix): void;
+74
View File
@@ -0,0 +1,74 @@
import {
FLAG,
getEmptyResult,
IOptions,
IResult,
parseImpl,
resetResult,
} from 'tldts-core';
import suffixLookup from './src/suffix-trie';
// For all methods but 'parse', it does not make sense to allocate an object
// every single time to only return the value of a specific attribute. To avoid
// this un-necessary allocation, we use a global object which is re-used.
const RESULT: IResult = getEmptyResult();
export function parse(url: string, options?: Partial<IOptions>): IResult {
return parseImpl(url, FLAG.ALL, suffixLookup, options, getEmptyResult());
}
export function getHostname(
url: string,
options?: Partial<IOptions>,
): string | null {
/*@__INLINE__*/ resetResult(RESULT);
return parseImpl(url, FLAG.HOSTNAME, suffixLookup, options, RESULT).hostname;
}
export function getPublicSuffix(
url: string,
options?: Partial<IOptions>,
): string | null {
/*@__INLINE__*/ resetResult(RESULT);
return parseImpl(url, FLAG.PUBLIC_SUFFIX, suffixLookup, options, RESULT)
.publicSuffix;
}
export function getDomain(
url: string,
options?: Partial<IOptions>,
): string | null {
/*@__INLINE__*/ resetResult(RESULT);
return parseImpl(url, FLAG.DOMAIN, suffixLookup, options, RESULT).domain;
}
export function getFullDomain(
url: string,
options?: Partial<IOptions>,
): string | null {
/*@__INLINE__*/ resetResult(RESULT);
const result = parseImpl(url, FLAG.DOMAIN, suffixLookup, options, RESULT);
// The hostname *is* the full domain (subdomain + domain) whenever a
// registrable domain exists; gate on `domain` so non-registrable inputs
// (IPs, suffix-less or invalid hostnames) return `null` like `getDomain`.
return result.domain === null ? null : result.hostname;
}
export function getSubdomain(
url: string,
options?: Partial<IOptions>,
): string | null {
/*@__INLINE__*/ resetResult(RESULT);
return parseImpl(url, FLAG.SUB_DOMAIN, suffixLookup, options, RESULT)
.subdomain;
}
export function getDomainWithoutSuffix(
url: string,
options?: Partial<IOptions>,
): string | null {
/*@__INLINE__*/ resetResult(RESULT);
return parseImpl(url, FLAG.ALL, suffixLookup, options, RESULT)
.domainWithoutSuffix;
}
+89
View File
@@ -0,0 +1,89 @@
{
"name": "tldts",
"version": "7.4.3",
"description": "Library to work against complex domain names, subdomains and URIs.",
"author": {
"name": "Rémi Berson"
},
"contributors": [
"Alexei <alexeiatyahoodotcom@gmail.com>",
"Alexey <kureev-mail@ya.ru>",
"Andrew <chefandrew@seomoz.org>",
"Johannes Ewald <johannes.ewald@peerigon.com>",
"Jérôme Desboeufs <jerome.desboeufs@gmail.com>",
"Kelly Campbell <kelly.a.campbell@gmail.com>",
"Kiko Beats <josefrancisco.verdu@gmail.com>",
"Kris Reeves <krisreeves@searchfanatics.com>",
"Krzysztof Jan Modras <chrmod@chrmod.net>",
"Olivier Melcher <olivier.melcher@gmail.com>",
"Rémi Berson <remi.berson@pm.me>",
"Saad Rashid <srashid@lendinghome.com>",
"Thomas Parisot <hi@oncletom.io>",
"Timo Tijhof <krinklemail@gmail.com>",
"Xavier Damman <xdamman@gmail.com>",
"Yehezkiel Syamsuhadi <yehezkielbs@gmail.com>"
],
"publishConfig": {
"access": "public"
},
"license": "MIT",
"homepage": "https://github.com/remusao/tldts#readme",
"bugs": {
"url": "https://github.com/remusao/tldts/issues"
},
"repository": {
"type": "git",
"url": "git+ssh://git@github.com/remusao/tldts.git"
},
"main": "dist/cjs/index.js",
"module": "dist/es6/index.js",
"types": "dist/types/index.d.ts",
"files": [
"dist",
"src",
"index.ts"
],
"bin": "bin/cli.js",
"scripts": {
"clean": "rimraf dist coverage",
"build": "tsc --build ./tsconfig.json",
"bundle": "tsc --build ./tsconfig.bundle.json && rollup --config ./rollup.config.mjs",
"prepack": "yarn run bundle",
"test": "nyc ../../node_modules/.bin/mocha --config ../../.mocharc.cjs"
},
"devDependencies": {
"@rollup/plugin-node-resolve": "^16.0.3",
"@rollup/plugin-terser": "^1.0.0",
"@rollup/plugin-typescript": "^12.3.0",
"@types/chai": "^5.2.3",
"@types/mocha": "^10.0.10",
"@types/node": "^25.1.0",
"chai": "^6.2.2",
"mocha": "^11.7.5",
"nyc": "^18.0.0",
"rimraf": "^6.1.2",
"rollup": "^4.57.1",
"rollup-plugin-sourcemaps2": "^0.5.4",
"tldts-tests": "^7.4.3",
"typescript": "^5.9.3"
},
"dependencies": {
"tldts-core": "^7.4.3"
},
"keywords": [
"tld",
"sld",
"domain",
"subdomain",
"subdomain",
"hostname",
"browser",
"uri",
"url",
"domain name",
"public suffix",
"url parsing",
"typescript"
],
"gitHead": "4447fc1a98b7566bdd2f7d054171aaeb261a17db"
}
+8
View File
File diff suppressed because one or more lines are too long
+207
View File
@@ -0,0 +1,207 @@
// NOTE: kept (intentionally) near-identical to packages/tldts-icann/src/suffix-trie.ts.
// They are separate copies rather than a shared helper because the lookup is
// only fast when the typed arrays are module-scope monomorphic globals —
// closing over them (a shared factory) measured ~20% slower. The ICANN build
// also specializes (constant mask, no isIcann/isPrivate). Keep the two in sync.
import {
fastPathLookup,
IPublicSuffix,
ISuffixLookupOptions,
} from 'tldts-core';
import {
edgeChild,
edgeLength,
edgeStart,
exceptionsRoot,
labelText,
nodeFlags,
rulesRoot,
} from './data/trie';
const enum RULE_TYPE {
ICANN = 1,
PRIVATE = 2,
}
// `edgeOffset` (where each label starts in `labelText`), `edgeHash` (djb2 of
// each label) and `wildcardEdge` (each node's '*' edge, or -1) are derived once
// at load instead of being shipped: the bundle then carries only the
// compressible `labelText` + structure, while the lookup binary-searches
// integer hashes. The cost is a single ~1ms pass at first import — cheaper than
// the object trie it replaces. Kept at module scope (not captured in a closure)
// so V8 treats the typed arrays as fast monomorphic globals.
const numberOfNodes = nodeFlags.length;
const numberOfEdges = edgeLength.length;
const edgeOffset = new Uint32Array(numberOfEdges);
const edgeHash = new Uint32Array(numberOfEdges);
const wildcardEdge = new Int32Array(numberOfNodes).fill(-1);
for (let node = 0, offset = 0; node < numberOfNodes; node += 1) {
for (let edge = edgeStart[node]!; edge < edgeStart[node + 1]!; edge += 1) {
edgeOffset[edge] = offset;
const end = offset + edgeLength[edge]!;
let hash = 5381;
for (let i = end - 1; i >= offset; i -= 1) {
hash = (hash * 33) ^ labelText.charCodeAt(i);
}
edgeHash[edge] = hash >>> 0;
if (
edgeLength[edge] === 1 &&
labelText.charCodeAt(offset) === 42 /* '*' */
) {
wildcardEdge[node] = edge;
}
offset = end;
}
}
// Result of the last `walk`, kept in module scope to avoid allocating a match
// object. Safe because lookups are synchronous and read right after `walk`.
let matchNode = -1;
let matchStart = 0;
let matchEnd = 0;
/**
* True if edge `edge`'s label equals `hostname[start, start + length)`.
*/
function labelEquals(
edge: number,
hostname: string,
start: number,
length: number,
): boolean {
if (edgeLength[edge] !== length) {
return false;
}
const offset = edgeOffset[edge]!;
for (let i = 0; i < length; i += 1) {
if (labelText.charCodeAt(offset + i) !== hostname.charCodeAt(start + i)) {
return false;
}
}
return true;
}
/**
* Find the child edge of `node` whose label is `hostname[start, start + length)`.
* Edges are sorted by hash, so binary-search the hash then verify the label
* (scanning the rare run of equal hashes). Returns the edge index or -1.
*/
function findEdge(
node: number,
hash: number,
hostname: string,
start: number,
length: number,
): number {
let lo = edgeStart[node]!;
let hi = edgeStart[node + 1]!;
while (lo < hi) {
const mid = (lo + hi) >>> 1;
const value = edgeHash[mid]!;
if (value < hash) {
lo = mid + 1;
} else if (value > hash) {
hi = mid;
} else {
for (let e = mid; e >= lo && edgeHash[e] === hash; e -= 1) {
if (labelEquals(e, hostname, start, length)) return e;
}
for (let e = mid + 1; e < hi && edgeHash[e] === hash; e += 1) {
if (labelEquals(e, hostname, start, length)) return e;
}
return -1;
}
}
return -1;
}
/**
* Walk `hostname`'s labels right-to-left from `root`, recording the deepest
* node whose flag passes `allowedMask` (with the label boundaries of that match
* in `matchStart`/`matchEnd`). Returns whether any match was found.
*/
function walk(hostname: string, root: number, allowedMask: number): boolean {
let node = root;
let end = hostname.length;
let hash = 5381;
matchNode = -1;
for (let i = hostname.length - 1; i >= 0; i -= 1) {
const code = hostname.charCodeAt(i);
if (code === 46 /* '.' */) {
const start = i + 1;
let edge = findEdge(node, hash >>> 0, hostname, start, end - start);
if (edge === -1) {
edge = wildcardEdge[node]!;
}
if (edge === -1) {
return matchNode !== -1;
}
node = edgeChild[edge]!;
if ((nodeFlags[node]! & allowedMask) !== 0) {
matchNode = node;
matchStart = start;
matchEnd = end;
}
end = i;
hash = 5381;
} else {
hash = (hash * 33) ^ code;
}
}
// Left-most label: hostname[0, end). Same find/descend/record as the loop —
// duplicated rather than folded into the loop (via `i >= -1`) because that
// extra per-character branch measured slightly slower on the hot path.
let edge = findEdge(node, hash >>> 0, hostname, 0, end);
if (edge === -1) {
edge = wildcardEdge[node]!;
}
if (edge !== -1) {
node = edgeChild[edge]!;
if ((nodeFlags[node]! & allowedMask) !== 0) {
matchNode = node;
matchStart = 0;
matchEnd = end;
}
}
return matchNode !== -1;
}
/**
* Check if `hostname` has a valid public suffix in the trie.
*/
export default function suffixLookup(
hostname: string,
options: ISuffixLookupOptions,
out: IPublicSuffix,
): void {
if (fastPathLookup(hostname, options, out)) {
return;
}
const allowedMask =
(options.allowPrivateDomains ? RULE_TYPE.PRIVATE : 0) |
(options.allowIcannDomains ? RULE_TYPE.ICANN : 0);
// Exceptions have priority and strip their own left-most label (e.g. the
// rule '!www.ck' makes the suffix of 'www.ck' be 'ck').
if (walk(hostname, exceptionsRoot, allowedMask)) {
out.isIcann = (nodeFlags[matchNode]! & RULE_TYPE.ICANN) !== 0;
out.isPrivate = (nodeFlags[matchNode]! & RULE_TYPE.PRIVATE) !== 0;
out.publicSuffix = hostname.slice(matchEnd + 1);
return;
}
if (walk(hostname, rulesRoot, allowedMask)) {
out.isIcann = (nodeFlags[matchNode]! & RULE_TYPE.ICANN) !== 0;
out.isPrivate = (nodeFlags[matchNode]! & RULE_TYPE.PRIVATE) !== 0;
out.publicSuffix = hostname.slice(matchStart);
return;
}
// No match: the prevailing '*' rule makes the right-most label the suffix.
out.isIcann = false;
out.isPrivate = false;
const lastDot = hostname.lastIndexOf('.');
out.publicSuffix = lastDot === -1 ? hostname : hostname.slice(lastDot + 1);
}