1
>
/*---------------------------------------------------------------------------------------------
textToHtmlTokenizer.ts
2
>
* Copyright (c) Microsoft Corporation. All rights reserved.
3
>
* Licensed under the MIT License. See License.txt in the project root for license information.
4
>
*--------------------------------------------------------------------------------------------*/
5
>
6
>
import { CharCode } from '../../../base/common/charCode.js';
7
>
import * as strings from '../../../base/common/strings.js';
8
>
import { IViewLineTokens, LineTokens } from '../tokens/lineTokens.js';
9
>
import { ILanguageIdCodec, IState, ITokenizationSupport, TokenizationRegistry } from '../languages.js';
10
>
import { LanguageId } from '../encodedTokenAttributes.js';
11
>
import { NullState, nullTokenizeEncoded } from './nullTokenize.js';
12
>
import { ILanguageService } from './language.js';
13
>
14
>
export type IReducedTokenizationSupport = Omit<ITokenizationSupport, 'tokenize'>;
15
>
16
>
const fallback: IReducedTokenizationSupport = {
17
>
getInitialState: () => NullState,
18
>
tokenizeEncoded: (buffer: string, hasEOL: boolean, state: IState) => nullTokenizeEncoded(LanguageId.Null, state)
19
>
};
20
>
21
>
export function tokenizeToStringSync(languageService: ILanguageService, text: string, languageId: string): string {
22
return _tokenizeToString(text, languageService.languageIdCodec, TokenizationRegistry.get(languageId) || fallback);
23
}
25
export async function tokenizeToString(languageService: ILanguageService, text: string, languageId: string | null): Promise<string> {
26
if (!languageId) {