更新
This commit is contained in:
+13
@@ -0,0 +1,13 @@
|
||||
Copyright (c) 2017 Thomas Parisot, 2018 Rémi Berson
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
+368
@@ -0,0 +1,368 @@
|
||||
# tldts - Blazing Fast URL Parsing
|
||||
|
||||
`tldts` is a JavaScript library to extract hostnames, domains, public suffixes, top-level domains and subdomains from URLs.
|
||||
|
||||
**Features**:
|
||||
|
||||
1. Tuned for **performance** (order of 0.1 to 1 μs per input)
|
||||
2. Handles both URLs and hostnames
|
||||
3. Full Unicode/IDNA support
|
||||
4. Support parsing email addresses
|
||||
5. Detect IPv4 and IPv6 addresses
|
||||
6. Continuously updated version of the public suffix list
|
||||
7. **TypeScript**, ships with `umd`, `esm`, `cjs` bundles and _type definitions_
|
||||
8. Small bundles and small memory footprint
|
||||
9. Battle tested: full test coverage and production use
|
||||
|
||||
# Install
|
||||
|
||||
```bash
|
||||
npm install --save tldts
|
||||
```
|
||||
|
||||
# Usage
|
||||
|
||||
Using the command-line interface:
|
||||
|
||||
```js
|
||||
$ npx tldts 'http://www.writethedocs.org/conf/eu/2017/'
|
||||
{
|
||||
"domain": "writethedocs.org",
|
||||
"domainWithoutSuffix": "writethedocs",
|
||||
"hostname": "www.writethedocs.org",
|
||||
"isIcann": true,
|
||||
"isIp": false,
|
||||
"isPrivate": false,
|
||||
"publicSuffix": "org",
|
||||
"subdomain": "www"
|
||||
}
|
||||
```
|
||||
|
||||
Programmatically:
|
||||
|
||||
```js
|
||||
const { parse } = require('tldts');
|
||||
|
||||
// Retrieving hostname related informations of a given URL
|
||||
parse('http://www.writethedocs.org/conf/eu/2017/');
|
||||
// { domain: 'writethedocs.org',
|
||||
// domainWithoutSuffix: 'writethedocs',
|
||||
// hostname: 'www.writethedocs.org',
|
||||
// isIcann: true,
|
||||
// isIp: false,
|
||||
// isPrivate: false,
|
||||
// publicSuffix: 'org',
|
||||
// subdomain: 'www' }
|
||||
```
|
||||
|
||||
Modern _ES6 modules import_ is also supported:
|
||||
|
||||
```js
|
||||
import { parse } from 'tldts';
|
||||
```
|
||||
|
||||
Alternatively, you can try it _directly in your browser_ here: https://npm.runkit.com/tldts
|
||||
|
||||
# API
|
||||
|
||||
- `tldts.parse(url | hostname, options)`
|
||||
- `tldts.getHostname(url | hostname, options)`
|
||||
- `tldts.getDomain(url | hostname, options)`
|
||||
- `tldts.getFullDomain(url | hostname, options)`
|
||||
- `tldts.getPublicSuffix(url | hostname, options)`
|
||||
- `tldts.getSubdomain(url, | hostname, options)`
|
||||
- `tldts.getDomainWithoutSuffix(url | hostname, options)`
|
||||
|
||||
The behavior of `tldts` can be customized using an `options` argument for all
|
||||
the functions exposed as part of the public API. This is useful to both change
|
||||
the behavior of the library as well as fine-tune the performance depending on
|
||||
your inputs.
|
||||
|
||||
```js
|
||||
{
|
||||
// Use suffixes from ICANN section (default: true)
|
||||
allowIcannDomains: boolean;
|
||||
// Use suffixes from Private section (default: false)
|
||||
allowPrivateDomains: boolean;
|
||||
// Extract and validate hostname (default: true)
|
||||
// When set to `false`, inputs will be considered valid hostnames.
|
||||
extractHostname: boolean;
|
||||
// Validate hostnames after parsing (default: true)
|
||||
// If a hostname is not valid, not further processing is performed. When set
|
||||
// to `false`, inputs to the library will be considered valid and parsing will
|
||||
// proceed regardless.
|
||||
validateHostname: boolean;
|
||||
// Perform IP address detection (default: true).
|
||||
detectIp: boolean;
|
||||
// Detect IANA special-use domains (RFC 6761 et al.) and expose the result as
|
||||
// `isSpecialUse` (default: false). Off by default so the common path does no
|
||||
// extra work; the field stays `null` unless this is enabled.
|
||||
detectSpecialUse: boolean;
|
||||
// Assume that both URLs and hostnames can be given as input (default: true)
|
||||
// If set to `false` we assume only URLs will be given as input, which
|
||||
// speed-ups processing.
|
||||
mixedInputs: boolean;
|
||||
// Specifies extra valid suffixes (default: null)
|
||||
validHosts: string[] | null;
|
||||
}
|
||||
```
|
||||
|
||||
The `parse` method returns handy **properties about a URL or a hostname**.
|
||||
|
||||
```js
|
||||
const tldts = require('tldts');
|
||||
|
||||
tldts.parse('https://spark-public.s3.amazonaws.com/dataanalysis/loansData.csv');
|
||||
// { domain: 'amazonaws.com',
|
||||
// domainWithoutSuffix: 'amazonaws',
|
||||
// hostname: 'spark-public.s3.amazonaws.com',
|
||||
// isIcann: true,
|
||||
// isIp: false,
|
||||
// isPrivate: false,
|
||||
// publicSuffix: 'com',
|
||||
// subdomain: 'spark-public.s3' }
|
||||
|
||||
tldts.parse(
|
||||
'https://spark-public.s3.amazonaws.com/dataanalysis/loansData.csv',
|
||||
{ allowPrivateDomains: true },
|
||||
);
|
||||
// { domain: 'spark-public.s3.amazonaws.com',
|
||||
// domainWithoutSuffix: 'spark-public',
|
||||
// hostname: 'spark-public.s3.amazonaws.com',
|
||||
// isIcann: false,
|
||||
// isIp: false,
|
||||
// isPrivate: true,
|
||||
// publicSuffix: 's3.amazonaws.com',
|
||||
// subdomain: '' }
|
||||
|
||||
tldts.parse('gopher://domain.unknown/');
|
||||
// { domain: 'domain.unknown',
|
||||
// domainWithoutSuffix: 'domain',
|
||||
// hostname: 'domain.unknown',
|
||||
// isIcann: false,
|
||||
// isIp: false,
|
||||
// isPrivate: false,
|
||||
// publicSuffix: 'unknown',
|
||||
// subdomain: '' }
|
||||
|
||||
tldts.parse('https://192.168.0.0'); // IPv4
|
||||
// { domain: null,
|
||||
// domainWithoutSuffix: null,
|
||||
// hostname: '192.168.0.0',
|
||||
// isIcann: null,
|
||||
// isIp: true,
|
||||
// isPrivate: null,
|
||||
// publicSuffix: null,
|
||||
// subdomain: null }
|
||||
|
||||
tldts.parse('https://[::1]'); // IPv6
|
||||
// { domain: null,
|
||||
// domainWithoutSuffix: null,
|
||||
// hostname: '::1',
|
||||
// isIcann: null,
|
||||
// isIp: true,
|
||||
// isPrivate: null,
|
||||
// publicSuffix: null,
|
||||
// subdomain: null }
|
||||
|
||||
tldts.parse('tldts@emailprovider.co.uk'); // email
|
||||
// { domain: 'emailprovider.co.uk',
|
||||
// domainWithoutSuffix: 'emailprovider',
|
||||
// hostname: 'emailprovider.co.uk',
|
||||
// isIcann: true,
|
||||
// isIp: false,
|
||||
// isPrivate: false,
|
||||
// publicSuffix: 'co.uk',
|
||||
// subdomain: '' }
|
||||
```
|
||||
|
||||
| Property Name | Type | Description |
|
||||
| :-------------------- | :----- | :---------------------------------------------- |
|
||||
| `hostname` | `str` | `hostname` of the input extracted automatically |
|
||||
| `domain` | `str` | Domain (tld + sld) |
|
||||
| `domainWithoutSuffix` | `str` | Domain without public suffix |
|
||||
| `subdomain` | `str` | Sub domain (what comes after `domain`) |
|
||||
| `publicSuffix` | `str` | Public Suffix (tld) of `hostname` |
|
||||
| `isIcann` | `bool` | Does TLD come from ICANN part of the list |
|
||||
| `isPrivate` | `bool` | Does TLD come from Private part of the list |
|
||||
| `isIP` | `bool` | Is `hostname` an IP address? |
|
||||
| `isSpecialUse` | `bool` | Is `hostname` an IANA special-use domain? |
|
||||
|
||||
## Special-use domains (RFC 6761 / IANA)
|
||||
|
||||
Set `{ detectSpecialUse: true }` to flag reserved special-use names such as `localhost`, `*.test`, `*.local`, `*.onion`, and `home.arpa` via the `isSpecialUse` result field. `isIcann`/`isPrivate` don't identify these: most aren't in the Public Suffix List, and the few that are (e.g. `onion`, `home.arpa`) appear there as ordinary ICANN suffixes. The field is `null` unless the option is enabled, so the default path does no extra work:
|
||||
|
||||
```js
|
||||
parse('http://printer.local/', { detectSpecialUse: true });
|
||||
// { ...
|
||||
// isSpecialUse: true,
|
||||
// publicSuffix: 'local',
|
||||
// subdomain: '' }
|
||||
```
|
||||
|
||||
The list tracks the IANA [Special-Use Domain Names](https://www.iana.org/assignments/special-use-domain-names/) registry.
|
||||
|
||||
## Single purpose methods
|
||||
|
||||
These methods are shorthands if you want to retrieve only a single value (and
|
||||
will perform better than `parse` because less work will be needed).
|
||||
|
||||
### getHostname(url | hostname, options?)
|
||||
|
||||
Returns the hostname from a given string.
|
||||
|
||||
```javascript
|
||||
const { getHostname } = require('tldts');
|
||||
|
||||
getHostname('google.com'); // returns `google.com`
|
||||
getHostname('fr.google.com'); // returns `fr.google.com`
|
||||
getHostname('fr.google.google'); // returns `fr.google.google`
|
||||
getHostname('foo.google.co.uk'); // returns `foo.google.co.uk`
|
||||
getHostname('t.co'); // returns `t.co`
|
||||
getHostname('fr.t.co'); // returns `fr.t.co`
|
||||
getHostname(
|
||||
'https://user:password@example.co.uk:8080/some/path?and&query#hash',
|
||||
); // returns `example.co.uk`
|
||||
```
|
||||
|
||||
### getDomain(url | hostname, options?)
|
||||
|
||||
Returns the fully qualified domain from a given string.
|
||||
|
||||
```javascript
|
||||
const { getDomain } = require('tldts');
|
||||
|
||||
getDomain('google.com'); // returns `google.com`
|
||||
getDomain('fr.google.com'); // returns `google.com`
|
||||
getDomain('fr.google.google'); // returns `google.google`
|
||||
getDomain('foo.google.co.uk'); // returns `google.co.uk`
|
||||
getDomain('t.co'); // returns `t.co`
|
||||
getDomain('fr.t.co'); // returns `t.co`
|
||||
getDomain('https://user:password@example.co.uk:8080/some/path?and&query#hash'); // returns `example.co.uk`
|
||||
```
|
||||
|
||||
### getFullDomain(url | hostname, options?)
|
||||
|
||||
Returns the full domain — the subdomain together with the registrable domain (as
|
||||
returned by `getDomain(...)`), i.e. the whole hostname _including_ any subdomain —
|
||||
or `null` when the input has no registrable domain (IP address, single label,
|
||||
bare public suffix, …). The result is the normalized hostname (lower-cased,
|
||||
trailing dot stripped); it is not a DNS-absolute name (no trailing root dot) and
|
||||
no IDNA/punycode conversion is performed.
|
||||
|
||||
```javascript
|
||||
const { getFullDomain } = require('tldts');
|
||||
|
||||
getFullDomain('google.com'); // returns `google.com`
|
||||
getFullDomain('fr.google.com'); // returns `fr.google.com`
|
||||
getFullDomain('foo.google.co.uk'); // returns `foo.google.co.uk`
|
||||
getFullDomain('t.co'); // returns `t.co`
|
||||
getFullDomain('fr.t.co'); // returns `fr.t.co`
|
||||
getFullDomain('1.2.3.4'); // returns null (no registrable domain)
|
||||
getFullDomain('localhost'); // returns null
|
||||
```
|
||||
|
||||
### getDomainWithoutSuffix(url | hostname, options?)
|
||||
|
||||
Returns the domain (as returned by `getDomain(...)`) without the public suffix part.
|
||||
|
||||
```javascript
|
||||
const { getDomainWithoutSuffix } = require('tldts');
|
||||
|
||||
getDomainWithoutSuffix('google.com'); // returns `google`
|
||||
getDomainWithoutSuffix('fr.google.com'); // returns `google`
|
||||
getDomainWithoutSuffix('fr.google.google'); // returns `google`
|
||||
getDomainWithoutSuffix('foo.google.co.uk'); // returns `google`
|
||||
getDomainWithoutSuffix('t.co'); // returns `t`
|
||||
getDomainWithoutSuffix('fr.t.co'); // returns `t`
|
||||
getDomainWithoutSuffix(
|
||||
'https://user:password@example.co.uk:8080/some/path?and&query#hash',
|
||||
); // returns `example`
|
||||
```
|
||||
|
||||
### getSubdomain(url | hostname, options?)
|
||||
|
||||
Returns the complete subdomain for a given string.
|
||||
|
||||
```javascript
|
||||
const { getSubdomain } = require('tldts');
|
||||
|
||||
getSubdomain('google.com'); // returns ``
|
||||
getSubdomain('fr.google.com'); // returns `fr`
|
||||
getSubdomain('google.co.uk'); // returns ``
|
||||
getSubdomain('foo.google.co.uk'); // returns `foo`
|
||||
getSubdomain('moar.foo.google.co.uk'); // returns `moar.foo`
|
||||
getSubdomain('t.co'); // returns ``
|
||||
getSubdomain('fr.t.co'); // returns `fr`
|
||||
getSubdomain(
|
||||
'https://user:password@secure.example.co.uk:443/some/path?and&query#hash',
|
||||
); // returns `secure`
|
||||
```
|
||||
|
||||
### getPublicSuffix(url | hostname, options?)
|
||||
|
||||
Returns the [public suffix][] for a given string.
|
||||
|
||||
```javascript
|
||||
const { getPublicSuffix } = require('tldts');
|
||||
|
||||
getPublicSuffix('google.com'); // returns `com`
|
||||
getPublicSuffix('fr.google.com'); // returns `com`
|
||||
getPublicSuffix('google.co.uk'); // returns `co.uk`
|
||||
getPublicSuffix('s3.amazonaws.com'); // returns `com`
|
||||
getPublicSuffix('s3.amazonaws.com', { allowPrivateDomains: true }); // returns `s3.amazonaws.com`
|
||||
getPublicSuffix('tld.is.unknown'); // returns `unknown`
|
||||
```
|
||||
|
||||
# Troubleshooting
|
||||
|
||||
## Retrieving subdomain of `localhost` and custom hostnames
|
||||
|
||||
`tldts` methods `getDomain` and `getSubdomain` are designed to **work only with _known and valid_ TLDs**.
|
||||
This way, you can trust what a domain is.
|
||||
|
||||
`localhost` is a valid hostname but not a TLD. You can pass additional options to each method exposed by `tldts`:
|
||||
|
||||
```js
|
||||
const tldts = require('tldts');
|
||||
|
||||
tldts.getDomain('localhost'); // returns null
|
||||
tldts.getSubdomain('vhost.localhost'); // returns null
|
||||
|
||||
tldts.getDomain('localhost', { validHosts: ['localhost'] }); // returns 'localhost'
|
||||
tldts.getSubdomain('vhost.localhost', { validHosts: ['localhost'] }); // returns 'vhost'
|
||||
```
|
||||
|
||||
## Updating the TLDs List
|
||||
|
||||
`tldts` made the opinionated choice of shipping with a list of suffixes directly
|
||||
in its bundle. There is currently no mechanism to update the lists yourself, but
|
||||
we make sure that the version shipped is always up-to-date.
|
||||
|
||||
If you keep `tldts` updated, the lists should be up-to-date as well!
|
||||
|
||||
# Performance
|
||||
|
||||
`tldts` is the _fastest JavaScript library_ available for parsing hostnames. It is able to parse _millions of inputs per second_ (typically 2-3M depending on your hardware and inputs). It also offers granular options to fine-tune the behavior and performance of the library depending on the kind of inputs you are dealing with (e.g.: if you know you only manipulate valid hostnames you can disable the hostname extraction step with `{ extractHostname: false }`).
|
||||
|
||||
Please see [this detailed comparison](./comparison/comparison.md) with other available libraries.
|
||||
|
||||
## Contributors
|
||||
|
||||
`tldts` is based upon the excellent `tld.js` library and would not exist without
|
||||
the many contributors who worked on the project:
|
||||
<a href="graphs/contributors"><img src="https://opencollective.com/tldjs/contributors.svg?width=890" /></a>
|
||||
|
||||
This project would not be possible without the amazing Mozilla's
|
||||
[public suffix list][]. Thank you for your hard work!
|
||||
|
||||
# License
|
||||
|
||||
[MIT License](LICENSE).
|
||||
|
||||
[badge-ci]: https://secure.travis-ci.org/remusao/tldts.svg?branch=master
|
||||
[badge-downloads]: https://img.shields.io/npm/dm/tldts.svg
|
||||
[public suffix list]: https://publicsuffix.org/list/
|
||||
[list the recent changes]: https://github.com/publicsuffix/list/commits/master
|
||||
[changes Atom Feed]: https://github.com/publicsuffix/list/commits/master.atom
|
||||
[public suffix]: https://publicsuffix.org/learn/
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
'use strict';
|
||||
|
||||
const { parse } = require('..');
|
||||
const readline = require('readline');
|
||||
|
||||
if (process.argv.length > 2) {
|
||||
// URL(s) was specified in the command arguments
|
||||
console.log(
|
||||
JSON.stringify(parse(process.argv[process.argv.length - 1]), null, 2),
|
||||
);
|
||||
} else {
|
||||
// No arguments were specified, read URLs from each line of STDIN
|
||||
const rlInterface = readline.createInterface({
|
||||
input: process.stdin,
|
||||
});
|
||||
rlInterface.on('line', function (line) {
|
||||
console.log(JSON.stringify(parse(line), null, 2));
|
||||
});
|
||||
}
|
||||
+1170
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+12
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+170
@@ -0,0 +1,170 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.default = suffixLookup;
|
||||
// NOTE: kept (intentionally) near-identical to packages/tldts-icann/src/suffix-trie.ts.
|
||||
// They are separate copies rather than a shared helper because the lookup is
|
||||
// only fast when the typed arrays are module-scope monomorphic globals —
|
||||
// closing over them (a shared factory) measured ~20% slower. The ICANN build
|
||||
// also specializes (constant mask, no isIcann/isPrivate). Keep the two in sync.
|
||||
const tldts_core_1 = require("tldts-core");
|
||||
const trie_1 = require("./data/trie");
|
||||
// `edgeOffset` (where each label starts in `labelText`), `edgeHash` (djb2 of
|
||||
// each label) and `wildcardEdge` (each node's '*' edge, or -1) are derived once
|
||||
// at load instead of being shipped: the bundle then carries only the
|
||||
// compressible `labelText` + structure, while the lookup binary-searches
|
||||
// integer hashes. The cost is a single ~1ms pass at first import — cheaper than
|
||||
// the object trie it replaces. Kept at module scope (not captured in a closure)
|
||||
// so V8 treats the typed arrays as fast monomorphic globals.
|
||||
const numberOfNodes = trie_1.nodeFlags.length;
|
||||
const numberOfEdges = trie_1.edgeLength.length;
|
||||
const edgeOffset = new Uint32Array(numberOfEdges);
|
||||
const edgeHash = new Uint32Array(numberOfEdges);
|
||||
const wildcardEdge = new Int32Array(numberOfNodes).fill(-1);
|
||||
for (let node = 0, offset = 0; node < numberOfNodes; node += 1) {
|
||||
for (let edge = trie_1.edgeStart[node]; edge < trie_1.edgeStart[node + 1]; edge += 1) {
|
||||
edgeOffset[edge] = offset;
|
||||
const end = offset + trie_1.edgeLength[edge];
|
||||
let hash = 5381;
|
||||
for (let i = end - 1; i >= offset; i -= 1) {
|
||||
hash = (hash * 33) ^ trie_1.labelText.charCodeAt(i);
|
||||
}
|
||||
edgeHash[edge] = hash >>> 0;
|
||||
if (trie_1.edgeLength[edge] === 1 &&
|
||||
trie_1.labelText.charCodeAt(offset) === 42 /* '*' */) {
|
||||
wildcardEdge[node] = edge;
|
||||
}
|
||||
offset = end;
|
||||
}
|
||||
}
|
||||
// Result of the last `walk`, kept in module scope to avoid allocating a match
|
||||
// object. Safe because lookups are synchronous and read right after `walk`.
|
||||
let matchNode = -1;
|
||||
let matchStart = 0;
|
||||
let matchEnd = 0;
|
||||
/**
|
||||
* True if edge `edge`'s label equals `hostname[start, start + length)`.
|
||||
*/
|
||||
function labelEquals(edge, hostname, start, length) {
|
||||
if (trie_1.edgeLength[edge] !== length) {
|
||||
return false;
|
||||
}
|
||||
const offset = edgeOffset[edge];
|
||||
for (let i = 0; i < length; i += 1) {
|
||||
if (trie_1.labelText.charCodeAt(offset + i) !== hostname.charCodeAt(start + i)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
/**
|
||||
* Find the child edge of `node` whose label is `hostname[start, start + length)`.
|
||||
* Edges are sorted by hash, so binary-search the hash then verify the label
|
||||
* (scanning the rare run of equal hashes). Returns the edge index or -1.
|
||||
*/
|
||||
function findEdge(node, hash, hostname, start, length) {
|
||||
let lo = trie_1.edgeStart[node];
|
||||
let hi = trie_1.edgeStart[node + 1];
|
||||
while (lo < hi) {
|
||||
const mid = (lo + hi) >>> 1;
|
||||
const value = edgeHash[mid];
|
||||
if (value < hash) {
|
||||
lo = mid + 1;
|
||||
}
|
||||
else if (value > hash) {
|
||||
hi = mid;
|
||||
}
|
||||
else {
|
||||
for (let e = mid; e >= lo && edgeHash[e] === hash; e -= 1) {
|
||||
if (labelEquals(e, hostname, start, length))
|
||||
return e;
|
||||
}
|
||||
for (let e = mid + 1; e < hi && edgeHash[e] === hash; e += 1) {
|
||||
if (labelEquals(e, hostname, start, length))
|
||||
return e;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
/**
|
||||
* Walk `hostname`'s labels right-to-left from `root`, recording the deepest
|
||||
* node whose flag passes `allowedMask` (with the label boundaries of that match
|
||||
* in `matchStart`/`matchEnd`). Returns whether any match was found.
|
||||
*/
|
||||
function walk(hostname, root, allowedMask) {
|
||||
let node = root;
|
||||
let end = hostname.length;
|
||||
let hash = 5381;
|
||||
matchNode = -1;
|
||||
for (let i = hostname.length - 1; i >= 0; i -= 1) {
|
||||
const code = hostname.charCodeAt(i);
|
||||
if (code === 46 /* '.' */) {
|
||||
const start = i + 1;
|
||||
let edge = findEdge(node, hash >>> 0, hostname, start, end - start);
|
||||
if (edge === -1) {
|
||||
edge = wildcardEdge[node];
|
||||
}
|
||||
if (edge === -1) {
|
||||
return matchNode !== -1;
|
||||
}
|
||||
node = trie_1.edgeChild[edge];
|
||||
if ((trie_1.nodeFlags[node] & allowedMask) !== 0) {
|
||||
matchNode = node;
|
||||
matchStart = start;
|
||||
matchEnd = end;
|
||||
}
|
||||
end = i;
|
||||
hash = 5381;
|
||||
}
|
||||
else {
|
||||
hash = (hash * 33) ^ code;
|
||||
}
|
||||
}
|
||||
// Left-most label: hostname[0, end). Same find/descend/record as the loop —
|
||||
// duplicated rather than folded into the loop (via `i >= -1`) because that
|
||||
// extra per-character branch measured slightly slower on the hot path.
|
||||
let edge = findEdge(node, hash >>> 0, hostname, 0, end);
|
||||
if (edge === -1) {
|
||||
edge = wildcardEdge[node];
|
||||
}
|
||||
if (edge !== -1) {
|
||||
node = trie_1.edgeChild[edge];
|
||||
if ((trie_1.nodeFlags[node] & allowedMask) !== 0) {
|
||||
matchNode = node;
|
||||
matchStart = 0;
|
||||
matchEnd = end;
|
||||
}
|
||||
}
|
||||
return matchNode !== -1;
|
||||
}
|
||||
/**
|
||||
* Check if `hostname` has a valid public suffix in the trie.
|
||||
*/
|
||||
function suffixLookup(hostname, options, out) {
|
||||
if ((0, tldts_core_1.fastPathLookup)(hostname, options, out)) {
|
||||
return;
|
||||
}
|
||||
const allowedMask = (options.allowPrivateDomains ? 2 /* RULE_TYPE.PRIVATE */ : 0) |
|
||||
(options.allowIcannDomains ? 1 /* RULE_TYPE.ICANN */ : 0);
|
||||
// Exceptions have priority and strip their own left-most label (e.g. the
|
||||
// rule '!www.ck' makes the suffix of 'www.ck' be 'ck').
|
||||
if (walk(hostname, trie_1.exceptionsRoot, allowedMask)) {
|
||||
out.isIcann = (trie_1.nodeFlags[matchNode] & 1 /* RULE_TYPE.ICANN */) !== 0;
|
||||
out.isPrivate = (trie_1.nodeFlags[matchNode] & 2 /* RULE_TYPE.PRIVATE */) !== 0;
|
||||
out.publicSuffix = hostname.slice(matchEnd + 1);
|
||||
return;
|
||||
}
|
||||
if (walk(hostname, trie_1.rulesRoot, allowedMask)) {
|
||||
out.isIcann = (trie_1.nodeFlags[matchNode] & 1 /* RULE_TYPE.ICANN */) !== 0;
|
||||
out.isPrivate = (trie_1.nodeFlags[matchNode] & 2 /* RULE_TYPE.PRIVATE */) !== 0;
|
||||
out.publicSuffix = hostname.slice(matchStart);
|
||||
return;
|
||||
}
|
||||
// No match: the prevailing '*' rule makes the right-most label the suffix.
|
||||
out.isIcann = false;
|
||||
out.isPrivate = false;
|
||||
const lastDot = hostname.lastIndexOf('.');
|
||||
out.publicSuffix = lastDot === -1 ? hostname : hostname.slice(lastDot + 1);
|
||||
}
|
||||
//# sourceMappingURL=suffix-trie.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+41
@@ -0,0 +1,41 @@
|
||||
import { getEmptyResult, parseImpl, resetResult, } from 'tldts-core';
|
||||
import suffixLookup from './src/suffix-trie';
|
||||
// For all methods but 'parse', it does not make sense to allocate an object
|
||||
// every single time to only return the value of a specific attribute. To avoid
|
||||
// this un-necessary allocation, we use a global object which is re-used.
|
||||
const RESULT = getEmptyResult();
|
||||
export function parse(url, options) {
|
||||
return parseImpl(url, 5 /* FLAG.ALL */, suffixLookup, options, getEmptyResult());
|
||||
}
|
||||
export function getHostname(url, options) {
|
||||
/*@__INLINE__*/ resetResult(RESULT);
|
||||
return parseImpl(url, 0 /* FLAG.HOSTNAME */, suffixLookup, options, RESULT).hostname;
|
||||
}
|
||||
export function getPublicSuffix(url, options) {
|
||||
/*@__INLINE__*/ resetResult(RESULT);
|
||||
return parseImpl(url, 2 /* FLAG.PUBLIC_SUFFIX */, suffixLookup, options, RESULT)
|
||||
.publicSuffix;
|
||||
}
|
||||
export function getDomain(url, options) {
|
||||
/*@__INLINE__*/ resetResult(RESULT);
|
||||
return parseImpl(url, 3 /* FLAG.DOMAIN */, suffixLookup, options, RESULT).domain;
|
||||
}
|
||||
export function getFullDomain(url, options) {
|
||||
/*@__INLINE__*/ resetResult(RESULT);
|
||||
const result = parseImpl(url, 3 /* FLAG.DOMAIN */, suffixLookup, options, RESULT);
|
||||
// The hostname *is* the full domain (subdomain + domain) whenever a
|
||||
// registrable domain exists; gate on `domain` so non-registrable inputs
|
||||
// (IPs, suffix-less or invalid hostnames) return `null` like `getDomain`.
|
||||
return result.domain === null ? null : result.hostname;
|
||||
}
|
||||
export function getSubdomain(url, options) {
|
||||
/*@__INLINE__*/ resetResult(RESULT);
|
||||
return parseImpl(url, 4 /* FLAG.SUB_DOMAIN */, suffixLookup, options, RESULT)
|
||||
.subdomain;
|
||||
}
|
||||
export function getDomainWithoutSuffix(url, options) {
|
||||
/*@__INLINE__*/ resetResult(RESULT);
|
||||
return parseImpl(url, 5 /* FLAG.ALL */, suffixLookup, options, RESULT)
|
||||
.domainWithoutSuffix;
|
||||
}
|
||||
//# sourceMappingURL=index.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../index.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,cAAc,EAGd,SAAS,EACT,WAAW,GACZ,MAAM,YAAY,CAAC;AAEpB,OAAO,YAAY,MAAM,mBAAmB,CAAC;AAE7C,4EAA4E;AAC5E,+EAA+E;AAC/E,yEAAyE;AACzE,MAAM,MAAM,GAAY,cAAc,EAAE,CAAC;AAEzC,MAAM,UAAU,KAAK,CAAC,GAAW,EAAE,OAA2B;IAC5D,OAAO,SAAS,CAAC,GAAG,oBAAY,YAAY,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,GAAW,EACX,OAA2B;IAE3B,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,SAAS,CAAC,GAAG,yBAAiB,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC;AAC/E,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,GAAW,EACX,OAA2B;IAE3B,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,SAAS,CAAC,GAAG,8BAAsB,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC;SACrE,YAAY,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,GAAW,EACX,OAA2B;IAE3B,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,SAAS,CAAC,GAAG,uBAAe,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC;AAC3E,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,GAAW,EACX,OAA2B;IAE3B,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,uBAAe,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC1E,oEAAoE;IACpE,wEAAwE;IACxE,0EAA0E;IAC1E,OAAO,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,GAAW,EACX,OAA2B;IAE3B,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,SAAS,CAAC,GAAG,2BAAmB,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC;SAClE,SAAS,CAAC;AACf,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,GAAW,EACX,OAA2B;IAE3B,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,SAAS,CAAC,GAAG,oBAAY,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC;SAC3D,mBAAmB,CAAC;AACzB,CAAC"}
|
||||
+9
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+167
@@ -0,0 +1,167 @@
|
||||
// NOTE: kept (intentionally) near-identical to packages/tldts-icann/src/suffix-trie.ts.
|
||||
// They are separate copies rather than a shared helper because the lookup is
|
||||
// only fast when the typed arrays are module-scope monomorphic globals —
|
||||
// closing over them (a shared factory) measured ~20% slower. The ICANN build
|
||||
// also specializes (constant mask, no isIcann/isPrivate). Keep the two in sync.
|
||||
import { fastPathLookup, } from 'tldts-core';
|
||||
import { edgeChild, edgeLength, edgeStart, exceptionsRoot, labelText, nodeFlags, rulesRoot, } from './data/trie';
|
||||
// `edgeOffset` (where each label starts in `labelText`), `edgeHash` (djb2 of
|
||||
// each label) and `wildcardEdge` (each node's '*' edge, or -1) are derived once
|
||||
// at load instead of being shipped: the bundle then carries only the
|
||||
// compressible `labelText` + structure, while the lookup binary-searches
|
||||
// integer hashes. The cost is a single ~1ms pass at first import — cheaper than
|
||||
// the object trie it replaces. Kept at module scope (not captured in a closure)
|
||||
// so V8 treats the typed arrays as fast monomorphic globals.
|
||||
const numberOfNodes = nodeFlags.length;
|
||||
const numberOfEdges = edgeLength.length;
|
||||
const edgeOffset = new Uint32Array(numberOfEdges);
|
||||
const edgeHash = new Uint32Array(numberOfEdges);
|
||||
const wildcardEdge = new Int32Array(numberOfNodes).fill(-1);
|
||||
for (let node = 0, offset = 0; node < numberOfNodes; node += 1) {
|
||||
for (let edge = edgeStart[node]; edge < edgeStart[node + 1]; edge += 1) {
|
||||
edgeOffset[edge] = offset;
|
||||
const end = offset + edgeLength[edge];
|
||||
let hash = 5381;
|
||||
for (let i = end - 1; i >= offset; i -= 1) {
|
||||
hash = (hash * 33) ^ labelText.charCodeAt(i);
|
||||
}
|
||||
edgeHash[edge] = hash >>> 0;
|
||||
if (edgeLength[edge] === 1 &&
|
||||
labelText.charCodeAt(offset) === 42 /* '*' */) {
|
||||
wildcardEdge[node] = edge;
|
||||
}
|
||||
offset = end;
|
||||
}
|
||||
}
|
||||
// Result of the last `walk`, kept in module scope to avoid allocating a match
|
||||
// object. Safe because lookups are synchronous and read right after `walk`.
|
||||
let matchNode = -1;
|
||||
let matchStart = 0;
|
||||
let matchEnd = 0;
|
||||
/**
|
||||
* True if edge `edge`'s label equals `hostname[start, start + length)`.
|
||||
*/
|
||||
function labelEquals(edge, hostname, start, length) {
|
||||
if (edgeLength[edge] !== length) {
|
||||
return false;
|
||||
}
|
||||
const offset = edgeOffset[edge];
|
||||
for (let i = 0; i < length; i += 1) {
|
||||
if (labelText.charCodeAt(offset + i) !== hostname.charCodeAt(start + i)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
/**
|
||||
* Find the child edge of `node` whose label is `hostname[start, start + length)`.
|
||||
* Edges are sorted by hash, so binary-search the hash then verify the label
|
||||
* (scanning the rare run of equal hashes). Returns the edge index or -1.
|
||||
*/
|
||||
function findEdge(node, hash, hostname, start, length) {
|
||||
let lo = edgeStart[node];
|
||||
let hi = edgeStart[node + 1];
|
||||
while (lo < hi) {
|
||||
const mid = (lo + hi) >>> 1;
|
||||
const value = edgeHash[mid];
|
||||
if (value < hash) {
|
||||
lo = mid + 1;
|
||||
}
|
||||
else if (value > hash) {
|
||||
hi = mid;
|
||||
}
|
||||
else {
|
||||
for (let e = mid; e >= lo && edgeHash[e] === hash; e -= 1) {
|
||||
if (labelEquals(e, hostname, start, length))
|
||||
return e;
|
||||
}
|
||||
for (let e = mid + 1; e < hi && edgeHash[e] === hash; e += 1) {
|
||||
if (labelEquals(e, hostname, start, length))
|
||||
return e;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
/**
|
||||
* Walk `hostname`'s labels right-to-left from `root`, recording the deepest
|
||||
* node whose flag passes `allowedMask` (with the label boundaries of that match
|
||||
* in `matchStart`/`matchEnd`). Returns whether any match was found.
|
||||
*/
|
||||
function walk(hostname, root, allowedMask) {
|
||||
let node = root;
|
||||
let end = hostname.length;
|
||||
let hash = 5381;
|
||||
matchNode = -1;
|
||||
for (let i = hostname.length - 1; i >= 0; i -= 1) {
|
||||
const code = hostname.charCodeAt(i);
|
||||
if (code === 46 /* '.' */) {
|
||||
const start = i + 1;
|
||||
let edge = findEdge(node, hash >>> 0, hostname, start, end - start);
|
||||
if (edge === -1) {
|
||||
edge = wildcardEdge[node];
|
||||
}
|
||||
if (edge === -1) {
|
||||
return matchNode !== -1;
|
||||
}
|
||||
node = edgeChild[edge];
|
||||
if ((nodeFlags[node] & allowedMask) !== 0) {
|
||||
matchNode = node;
|
||||
matchStart = start;
|
||||
matchEnd = end;
|
||||
}
|
||||
end = i;
|
||||
hash = 5381;
|
||||
}
|
||||
else {
|
||||
hash = (hash * 33) ^ code;
|
||||
}
|
||||
}
|
||||
// Left-most label: hostname[0, end). Same find/descend/record as the loop —
|
||||
// duplicated rather than folded into the loop (via `i >= -1`) because that
|
||||
// extra per-character branch measured slightly slower on the hot path.
|
||||
let edge = findEdge(node, hash >>> 0, hostname, 0, end);
|
||||
if (edge === -1) {
|
||||
edge = wildcardEdge[node];
|
||||
}
|
||||
if (edge !== -1) {
|
||||
node = edgeChild[edge];
|
||||
if ((nodeFlags[node] & allowedMask) !== 0) {
|
||||
matchNode = node;
|
||||
matchStart = 0;
|
||||
matchEnd = end;
|
||||
}
|
||||
}
|
||||
return matchNode !== -1;
|
||||
}
|
||||
/**
|
||||
* Check if `hostname` has a valid public suffix in the trie.
|
||||
*/
|
||||
export default function suffixLookup(hostname, options, out) {
|
||||
if (fastPathLookup(hostname, options, out)) {
|
||||
return;
|
||||
}
|
||||
const allowedMask = (options.allowPrivateDomains ? 2 /* RULE_TYPE.PRIVATE */ : 0) |
|
||||
(options.allowIcannDomains ? 1 /* RULE_TYPE.ICANN */ : 0);
|
||||
// Exceptions have priority and strip their own left-most label (e.g. the
|
||||
// rule '!www.ck' makes the suffix of 'www.ck' be 'ck').
|
||||
if (walk(hostname, exceptionsRoot, allowedMask)) {
|
||||
out.isIcann = (nodeFlags[matchNode] & 1 /* RULE_TYPE.ICANN */) !== 0;
|
||||
out.isPrivate = (nodeFlags[matchNode] & 2 /* RULE_TYPE.PRIVATE */) !== 0;
|
||||
out.publicSuffix = hostname.slice(matchEnd + 1);
|
||||
return;
|
||||
}
|
||||
if (walk(hostname, rulesRoot, allowedMask)) {
|
||||
out.isIcann = (nodeFlags[matchNode] & 1 /* RULE_TYPE.ICANN */) !== 0;
|
||||
out.isPrivate = (nodeFlags[matchNode] & 2 /* RULE_TYPE.PRIVATE */) !== 0;
|
||||
out.publicSuffix = hostname.slice(matchStart);
|
||||
return;
|
||||
}
|
||||
// No match: the prevailing '*' rule makes the right-most label the suffix.
|
||||
out.isIcann = false;
|
||||
out.isPrivate = false;
|
||||
const lastDot = hostname.lastIndexOf('.');
|
||||
out.publicSuffix = lastDot === -1 ? hostname : hostname.slice(lastDot + 1);
|
||||
}
|
||||
//# sourceMappingURL=suffix-trie.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+2
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+2
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+2
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+8
@@ -0,0 +1,8 @@
|
||||
import { IOptions, IResult } from 'tldts-core';
|
||||
export declare function parse(url: string, options?: Partial<IOptions>): IResult;
|
||||
export declare function getHostname(url: string, options?: Partial<IOptions>): string | null;
|
||||
export declare function getPublicSuffix(url: string, options?: Partial<IOptions>): string | null;
|
||||
export declare function getDomain(url: string, options?: Partial<IOptions>): string | null;
|
||||
export declare function getFullDomain(url: string, options?: Partial<IOptions>): string | null;
|
||||
export declare function getSubdomain(url: string, options?: Partial<IOptions>): string | null;
|
||||
export declare function getDomainWithoutSuffix(url: string, options?: Partial<IOptions>): string | null;
|
||||
+7
File diff suppressed because one or more lines are too long
+5
@@ -0,0 +1,5 @@
|
||||
import { IPublicSuffix, ISuffixLookupOptions } from 'tldts-core';
|
||||
/**
|
||||
* Check if `hostname` has a valid public suffix in the trie.
|
||||
*/
|
||||
export default function suffixLookup(hostname: string, options: ISuffixLookupOptions, out: IPublicSuffix): void;
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
import {
|
||||
FLAG,
|
||||
getEmptyResult,
|
||||
IOptions,
|
||||
IResult,
|
||||
parseImpl,
|
||||
resetResult,
|
||||
} from 'tldts-core';
|
||||
|
||||
import suffixLookup from './src/suffix-trie';
|
||||
|
||||
// For all methods but 'parse', it does not make sense to allocate an object
|
||||
// every single time to only return the value of a specific attribute. To avoid
|
||||
// this un-necessary allocation, we use a global object which is re-used.
|
||||
const RESULT: IResult = getEmptyResult();
|
||||
|
||||
export function parse(url: string, options?: Partial<IOptions>): IResult {
|
||||
return parseImpl(url, FLAG.ALL, suffixLookup, options, getEmptyResult());
|
||||
}
|
||||
|
||||
export function getHostname(
|
||||
url: string,
|
||||
options?: Partial<IOptions>,
|
||||
): string | null {
|
||||
/*@__INLINE__*/ resetResult(RESULT);
|
||||
return parseImpl(url, FLAG.HOSTNAME, suffixLookup, options, RESULT).hostname;
|
||||
}
|
||||
|
||||
export function getPublicSuffix(
|
||||
url: string,
|
||||
options?: Partial<IOptions>,
|
||||
): string | null {
|
||||
/*@__INLINE__*/ resetResult(RESULT);
|
||||
return parseImpl(url, FLAG.PUBLIC_SUFFIX, suffixLookup, options, RESULT)
|
||||
.publicSuffix;
|
||||
}
|
||||
|
||||
export function getDomain(
|
||||
url: string,
|
||||
options?: Partial<IOptions>,
|
||||
): string | null {
|
||||
/*@__INLINE__*/ resetResult(RESULT);
|
||||
return parseImpl(url, FLAG.DOMAIN, suffixLookup, options, RESULT).domain;
|
||||
}
|
||||
|
||||
export function getFullDomain(
|
||||
url: string,
|
||||
options?: Partial<IOptions>,
|
||||
): string | null {
|
||||
/*@__INLINE__*/ resetResult(RESULT);
|
||||
const result = parseImpl(url, FLAG.DOMAIN, suffixLookup, options, RESULT);
|
||||
// The hostname *is* the full domain (subdomain + domain) whenever a
|
||||
// registrable domain exists; gate on `domain` so non-registrable inputs
|
||||
// (IPs, suffix-less or invalid hostnames) return `null` like `getDomain`.
|
||||
return result.domain === null ? null : result.hostname;
|
||||
}
|
||||
|
||||
export function getSubdomain(
|
||||
url: string,
|
||||
options?: Partial<IOptions>,
|
||||
): string | null {
|
||||
/*@__INLINE__*/ resetResult(RESULT);
|
||||
return parseImpl(url, FLAG.SUB_DOMAIN, suffixLookup, options, RESULT)
|
||||
.subdomain;
|
||||
}
|
||||
|
||||
export function getDomainWithoutSuffix(
|
||||
url: string,
|
||||
options?: Partial<IOptions>,
|
||||
): string | null {
|
||||
/*@__INLINE__*/ resetResult(RESULT);
|
||||
return parseImpl(url, FLAG.ALL, suffixLookup, options, RESULT)
|
||||
.domainWithoutSuffix;
|
||||
}
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
{
|
||||
"name": "tldts",
|
||||
"version": "7.4.3",
|
||||
"description": "Library to work against complex domain names, subdomains and URIs.",
|
||||
"author": {
|
||||
"name": "Rémi Berson"
|
||||
},
|
||||
"contributors": [
|
||||
"Alexei <alexeiatyahoodotcom@gmail.com>",
|
||||
"Alexey <kureev-mail@ya.ru>",
|
||||
"Andrew <chefandrew@seomoz.org>",
|
||||
"Johannes Ewald <johannes.ewald@peerigon.com>",
|
||||
"Jérôme Desboeufs <jerome.desboeufs@gmail.com>",
|
||||
"Kelly Campbell <kelly.a.campbell@gmail.com>",
|
||||
"Kiko Beats <josefrancisco.verdu@gmail.com>",
|
||||
"Kris Reeves <krisreeves@searchfanatics.com>",
|
||||
"Krzysztof Jan Modras <chrmod@chrmod.net>",
|
||||
"Olivier Melcher <olivier.melcher@gmail.com>",
|
||||
"Rémi Berson <remi.berson@pm.me>",
|
||||
"Saad Rashid <srashid@lendinghome.com>",
|
||||
"Thomas Parisot <hi@oncletom.io>",
|
||||
"Timo Tijhof <krinklemail@gmail.com>",
|
||||
"Xavier Damman <xdamman@gmail.com>",
|
||||
"Yehezkiel Syamsuhadi <yehezkielbs@gmail.com>"
|
||||
],
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
"license": "MIT",
|
||||
"homepage": "https://github.com/remusao/tldts#readme",
|
||||
"bugs": {
|
||||
"url": "https://github.com/remusao/tldts/issues"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+ssh://git@github.com/remusao/tldts.git"
|
||||
},
|
||||
"main": "dist/cjs/index.js",
|
||||
"module": "dist/es6/index.js",
|
||||
"types": "dist/types/index.d.ts",
|
||||
"files": [
|
||||
"dist",
|
||||
"src",
|
||||
"index.ts"
|
||||
],
|
||||
"bin": "bin/cli.js",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist coverage",
|
||||
"build": "tsc --build ./tsconfig.json",
|
||||
"bundle": "tsc --build ./tsconfig.bundle.json && rollup --config ./rollup.config.mjs",
|
||||
"prepack": "yarn run bundle",
|
||||
"test": "nyc ../../node_modules/.bin/mocha --config ../../.mocharc.cjs"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@rollup/plugin-node-resolve": "^16.0.3",
|
||||
"@rollup/plugin-terser": "^1.0.0",
|
||||
"@rollup/plugin-typescript": "^12.3.0",
|
||||
"@types/chai": "^5.2.3",
|
||||
"@types/mocha": "^10.0.10",
|
||||
"@types/node": "^25.1.0",
|
||||
"chai": "^6.2.2",
|
||||
"mocha": "^11.7.5",
|
||||
"nyc": "^18.0.0",
|
||||
"rimraf": "^6.1.2",
|
||||
"rollup": "^4.57.1",
|
||||
"rollup-plugin-sourcemaps2": "^0.5.4",
|
||||
"tldts-tests": "^7.4.3",
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"tldts-core": "^7.4.3"
|
||||
},
|
||||
"keywords": [
|
||||
"tld",
|
||||
"sld",
|
||||
"domain",
|
||||
"subdomain",
|
||||
"subdomain",
|
||||
"hostname",
|
||||
"browser",
|
||||
"uri",
|
||||
"url",
|
||||
"domain name",
|
||||
"public suffix",
|
||||
"url parsing",
|
||||
"typescript"
|
||||
],
|
||||
"gitHead": "4447fc1a98b7566bdd2f7d054171aaeb261a17db"
|
||||
}
|
||||
+8
File diff suppressed because one or more lines are too long
+207
@@ -0,0 +1,207 @@
|
||||
// NOTE: kept (intentionally) near-identical to packages/tldts-icann/src/suffix-trie.ts.
|
||||
// They are separate copies rather than a shared helper because the lookup is
|
||||
// only fast when the typed arrays are module-scope monomorphic globals —
|
||||
// closing over them (a shared factory) measured ~20% slower. The ICANN build
|
||||
// also specializes (constant mask, no isIcann/isPrivate). Keep the two in sync.
|
||||
import {
|
||||
fastPathLookup,
|
||||
IPublicSuffix,
|
||||
ISuffixLookupOptions,
|
||||
} from 'tldts-core';
|
||||
import {
|
||||
edgeChild,
|
||||
edgeLength,
|
||||
edgeStart,
|
||||
exceptionsRoot,
|
||||
labelText,
|
||||
nodeFlags,
|
||||
rulesRoot,
|
||||
} from './data/trie';
|
||||
|
||||
const enum RULE_TYPE {
|
||||
ICANN = 1,
|
||||
PRIVATE = 2,
|
||||
}
|
||||
|
||||
// `edgeOffset` (where each label starts in `labelText`), `edgeHash` (djb2 of
|
||||
// each label) and `wildcardEdge` (each node's '*' edge, or -1) are derived once
|
||||
// at load instead of being shipped: the bundle then carries only the
|
||||
// compressible `labelText` + structure, while the lookup binary-searches
|
||||
// integer hashes. The cost is a single ~1ms pass at first import — cheaper than
|
||||
// the object trie it replaces. Kept at module scope (not captured in a closure)
|
||||
// so V8 treats the typed arrays as fast monomorphic globals.
|
||||
const numberOfNodes = nodeFlags.length;
|
||||
const numberOfEdges = edgeLength.length;
|
||||
const edgeOffset = new Uint32Array(numberOfEdges);
|
||||
const edgeHash = new Uint32Array(numberOfEdges);
|
||||
const wildcardEdge = new Int32Array(numberOfNodes).fill(-1);
|
||||
for (let node = 0, offset = 0; node < numberOfNodes; node += 1) {
|
||||
for (let edge = edgeStart[node]!; edge < edgeStart[node + 1]!; edge += 1) {
|
||||
edgeOffset[edge] = offset;
|
||||
const end = offset + edgeLength[edge]!;
|
||||
let hash = 5381;
|
||||
for (let i = end - 1; i >= offset; i -= 1) {
|
||||
hash = (hash * 33) ^ labelText.charCodeAt(i);
|
||||
}
|
||||
edgeHash[edge] = hash >>> 0;
|
||||
if (
|
||||
edgeLength[edge] === 1 &&
|
||||
labelText.charCodeAt(offset) === 42 /* '*' */
|
||||
) {
|
||||
wildcardEdge[node] = edge;
|
||||
}
|
||||
offset = end;
|
||||
}
|
||||
}
|
||||
|
||||
// Result of the last `walk`, kept in module scope to avoid allocating a match
|
||||
// object. Safe because lookups are synchronous and read right after `walk`.
|
||||
let matchNode = -1;
|
||||
let matchStart = 0;
|
||||
let matchEnd = 0;
|
||||
|
||||
/**
|
||||
* True if edge `edge`'s label equals `hostname[start, start + length)`.
|
||||
*/
|
||||
function labelEquals(
|
||||
edge: number,
|
||||
hostname: string,
|
||||
start: number,
|
||||
length: number,
|
||||
): boolean {
|
||||
if (edgeLength[edge] !== length) {
|
||||
return false;
|
||||
}
|
||||
const offset = edgeOffset[edge]!;
|
||||
for (let i = 0; i < length; i += 1) {
|
||||
if (labelText.charCodeAt(offset + i) !== hostname.charCodeAt(start + i)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the child edge of `node` whose label is `hostname[start, start + length)`.
|
||||
* Edges are sorted by hash, so binary-search the hash then verify the label
|
||||
* (scanning the rare run of equal hashes). Returns the edge index or -1.
|
||||
*/
|
||||
function findEdge(
|
||||
node: number,
|
||||
hash: number,
|
||||
hostname: string,
|
||||
start: number,
|
||||
length: number,
|
||||
): number {
|
||||
let lo = edgeStart[node]!;
|
||||
let hi = edgeStart[node + 1]!;
|
||||
while (lo < hi) {
|
||||
const mid = (lo + hi) >>> 1;
|
||||
const value = edgeHash[mid]!;
|
||||
if (value < hash) {
|
||||
lo = mid + 1;
|
||||
} else if (value > hash) {
|
||||
hi = mid;
|
||||
} else {
|
||||
for (let e = mid; e >= lo && edgeHash[e] === hash; e -= 1) {
|
||||
if (labelEquals(e, hostname, start, length)) return e;
|
||||
}
|
||||
for (let e = mid + 1; e < hi && edgeHash[e] === hash; e += 1) {
|
||||
if (labelEquals(e, hostname, start, length)) return e;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk `hostname`'s labels right-to-left from `root`, recording the deepest
|
||||
* node whose flag passes `allowedMask` (with the label boundaries of that match
|
||||
* in `matchStart`/`matchEnd`). Returns whether any match was found.
|
||||
*/
|
||||
function walk(hostname: string, root: number, allowedMask: number): boolean {
|
||||
let node = root;
|
||||
let end = hostname.length;
|
||||
let hash = 5381;
|
||||
matchNode = -1;
|
||||
for (let i = hostname.length - 1; i >= 0; i -= 1) {
|
||||
const code = hostname.charCodeAt(i);
|
||||
if (code === 46 /* '.' */) {
|
||||
const start = i + 1;
|
||||
let edge = findEdge(node, hash >>> 0, hostname, start, end - start);
|
||||
if (edge === -1) {
|
||||
edge = wildcardEdge[node]!;
|
||||
}
|
||||
if (edge === -1) {
|
||||
return matchNode !== -1;
|
||||
}
|
||||
node = edgeChild[edge]!;
|
||||
if ((nodeFlags[node]! & allowedMask) !== 0) {
|
||||
matchNode = node;
|
||||
matchStart = start;
|
||||
matchEnd = end;
|
||||
}
|
||||
end = i;
|
||||
hash = 5381;
|
||||
} else {
|
||||
hash = (hash * 33) ^ code;
|
||||
}
|
||||
}
|
||||
|
||||
// Left-most label: hostname[0, end). Same find/descend/record as the loop —
|
||||
// duplicated rather than folded into the loop (via `i >= -1`) because that
|
||||
// extra per-character branch measured slightly slower on the hot path.
|
||||
let edge = findEdge(node, hash >>> 0, hostname, 0, end);
|
||||
if (edge === -1) {
|
||||
edge = wildcardEdge[node]!;
|
||||
}
|
||||
if (edge !== -1) {
|
||||
node = edgeChild[edge]!;
|
||||
if ((nodeFlags[node]! & allowedMask) !== 0) {
|
||||
matchNode = node;
|
||||
matchStart = 0;
|
||||
matchEnd = end;
|
||||
}
|
||||
}
|
||||
return matchNode !== -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if `hostname` has a valid public suffix in the trie.
|
||||
*/
|
||||
export default function suffixLookup(
|
||||
hostname: string,
|
||||
options: ISuffixLookupOptions,
|
||||
out: IPublicSuffix,
|
||||
): void {
|
||||
if (fastPathLookup(hostname, options, out)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const allowedMask =
|
||||
(options.allowPrivateDomains ? RULE_TYPE.PRIVATE : 0) |
|
||||
(options.allowIcannDomains ? RULE_TYPE.ICANN : 0);
|
||||
|
||||
// Exceptions have priority and strip their own left-most label (e.g. the
|
||||
// rule '!www.ck' makes the suffix of 'www.ck' be 'ck').
|
||||
if (walk(hostname, exceptionsRoot, allowedMask)) {
|
||||
out.isIcann = (nodeFlags[matchNode]! & RULE_TYPE.ICANN) !== 0;
|
||||
out.isPrivate = (nodeFlags[matchNode]! & RULE_TYPE.PRIVATE) !== 0;
|
||||
out.publicSuffix = hostname.slice(matchEnd + 1);
|
||||
return;
|
||||
}
|
||||
|
||||
if (walk(hostname, rulesRoot, allowedMask)) {
|
||||
out.isIcann = (nodeFlags[matchNode]! & RULE_TYPE.ICANN) !== 0;
|
||||
out.isPrivate = (nodeFlags[matchNode]! & RULE_TYPE.PRIVATE) !== 0;
|
||||
out.publicSuffix = hostname.slice(matchStart);
|
||||
return;
|
||||
}
|
||||
|
||||
// No match: the prevailing '*' rule makes the right-most label the suffix.
|
||||
out.isIcann = false;
|
||||
out.isPrivate = false;
|
||||
const lastDot = hostname.lastIndexOf('.');
|
||||
out.publicSuffix = lastDot === -1 ? hostname : hostname.slice(lastDot + 1);
|
||||
}
|
||||
Reference in New Issue
Block a user