1
>
/*---------------------------------------------------------------------------------------------
encoding.ts
2
>
* Copyright (c) Microsoft Corporation. All rights reserved.
3
>
* Licensed under the MIT License. See License.txt in the project root for license information.
4
>
*--------------------------------------------------------------------------------------------*/
5
>
6
>
import { Readable, ReadableStream, newWriteableStream, listenStream } from '../../../../base/common/stream.js';
7
>
import { VSBuffer, VSBufferReadable, VSBufferReadableStream } from '../../../../base/common/buffer.js';
8
>
import { importAMDNodeModule } from '../../../../amdX.js';
9
>
import { CancellationTokenSource } from '../../../../base/common/cancellation.js';
10
>
import { coalesce } from '../../../../base/common/arrays.js';
11
>
12
>
export const UTF8 = 'utf8';
13
>
export const UTF8_with_bom = 'utf8bom';
14
>
export const UTF16be = 'utf16be';
15
>
export const UTF16le = 'utf16le';
16
>
17
>
export type UTF_ENCODING = typeof UTF8 | typeof UTF8_with_bom | typeof UTF16be | typeof UTF16le;
18
>
19
>
export function isUTFEncoding(encoding: string): encoding is UTF_ENCODING {
20
return [UTF8, UTF8_with_bom, UTF16be, UTF16le].some(utfEncoding => utfEncoding === encoding);
21
}
23
>
export const UTF16be_BOM = [0xFE, 0xFF];
24
>
export const UTF16le_BOM = [0xFF, 0xFE];
25
>
export const UTF8_BOM = [0xEF, 0xBB, 0xBF];
26
>
27
>
const ZERO_BYTE_DETECTION_BUFFER_MAX_LEN = 512; // number of bytes to look at to decide about a file being binary or not
28
>
const NO_ENCODING_GUESS_MIN_BYTES = 512; // when not auto guessing the encoding, small number of bytes are enough
29
>
const AUTO_ENCODING_GUESS_MIN_BYTES = 512 * 8; // with auto guessing we want a lot more content to be read for guessing
30
>
const AUTO_ENCODING_GUESS_MAX_BYTES = 512 * 128; // set an upper limit for the number of bytes we pass on to jschardet
31
>
32
>
export interface IDecodeStreamOptions {
33
>
acceptTextOnly: boolean;
34
>
guessEncoding: boolean;
35
>
candidateGuessEncodings: string[];
36
>
minBytesRequiredForDetection?: number;
37
>
38
>
overwriteEncoding(detectedEncoding: string | null): Promise<string>;
39
>
}
40
>
41
>
export interface IDecodeStreamResult {
42
>
stream: ReadableStream<string>;
43
>
detected: IDetectedEncodingResult;
44
>
}
45
>
46
>
export const enum DecodeStreamErrorKind {
47
>
48
>
/**
49
>
* Error indicating that the stream is binary even
50
>
* though `acceptTextOnly` was specified.
51
>
*/
52
>
STREAM_IS_BINARY = 1
53
>
}
54
>
55
>
export class DecodeStreamError extends Error {
56
>
57
>
constructor(
58
message: string,
59
readonly decodeStreamErrorKind: DecodeStreamErrorKind