Class: TokenTextSplitter

text_splitter.TokenTextSplitter

Implementation of splitter which looks at tokens.

Hierarchy

• TextSplitter

  ↳ TokenTextSplitter

Implements

• TokenTextSplitterParams

Constructors

constructor

• new TokenTextSplitter(fields?)

Parameters

Name	Type
fields?	Partial<TokenTextSplitterParams>

Overrides

TextSplitter.constructor

Defined in

langchain/src/text_splitter.ts:244

Properties

allowedSpecial

• allowedSpecial: string[] | "all"

Implementation of

TokenTextSplitterParams.allowedSpecial

Defined in

langchain/src/text_splitter.ts:236

---

chunkOverlap

• chunkOverlap: number = 200

Implementation of

TokenTextSplitterParams.chunkOverlap

Inherited from

TextSplitter.chunkOverlap

Defined in

langchain/src/text_splitter.ts:13

---

chunkSize

• chunkSize: number = 1000

Implementation of

TokenTextSplitterParams.chunkSize

Inherited from

TextSplitter.chunkSize

Defined in

langchain/src/text_splitter.ts:11

---

disallowedSpecial

• disallowedSpecial: string[] | "all"

Implementation of

TokenTextSplitterParams.disallowedSpecial

Defined in

langchain/src/text_splitter.ts:238

---

encodingName

• encodingName: TiktokenEncoding

Implementation of

TokenTextSplitterParams.encodingName

Defined in

langchain/src/text_splitter.ts:234

Methods

createDocuments

▸ createDocuments(texts, metadatas?): Promise<Document<Record<string, any>>[]>

Parameters

Name	Type	Default value
texts	string[]	undefined
metadatas	Record<string, any>[]	[]

Returns

Promise<Document<Record<string, any>>[]>

Inherited from

TextSplitter.createDocuments

Defined in

langchain/src/text_splitter.ts:25

---

mergeSplits

▸ mergeSplits(splits, separator): string[]

Parameters

Name	Type
splits	string[]
separator	string

Returns

string[]

Inherited from

TextSplitter.mergeSplits

Defined in

langchain/src/text_splitter.ts:90

---

splitDocuments

▸ splitDocuments(documents): Promise<Document<Record<string, any>>[]>

Parameters

Name	Type
documents	Document<Record<string, any>>[]

Returns

Promise<Document<Record<string, any>>[]>

Inherited from

TextSplitter.splitDocuments

Defined in

langchain/src/text_splitter.ts:79

---

splitText

▸ splitText(text): Promise<string[]>

Parameters

Name	Type
text	string

Returns

Promise<string[]>

Overrides

TextSplitter.splitText

Defined in

langchain/src/text_splitter.ts:252

---

imports

▸ Static imports(): Promise<__module>

Returns

Promise<__module>

Defined in

langchain/src/text_splitter.ts:287