Skip to content

Commit 0819a7f

Browse files
pythoninofficepythoninoffice
authored
Add files via upload
1 parent 3410e9e commit 0819a7f

1 file changed

Lines changed: 297 additions & 0 deletions

File tree

‎langchain_examples/Langchain_token_limit_chain_type.ipynb‎

Lines changed: 297 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,297 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"id": "45e39a54-412d-4c74-9927-4d0d1ce622dc",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"from dotenv import dotenv_values\n",
11+
"from langchain.chains.summarize import load_summarize_chain\n",
12+
"from langchain.document_loaders import TextLoader\n",
13+
"from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
14+
"from langchain.llms import OpenAI\n",
15+
"import os"
16+
]
17+
},
18+
{
19+
"cell_type": "code",
20+
"execution_count": null,
21+
"id": "a3ce13d4-31d9-403b-a3a5-c7ffee05f65b",
22+
"metadata": {},
23+
"outputs": [],
24+
"source": [
25+
"api_keys=dotenv_values()\n",
26+
"os.environ['OPENAI_API_KEY'] = dotenv_values()['openai_api_key'] #set environment variable"
27+
]
28+
},
29+
{
30+
"cell_type": "code",
31+
"execution_count": null,
32+
"id": "95f07628-70f8-41f7-9f1d-d49a404cb2d4",
33+
"metadata": {},
34+
"outputs": [],
35+
"source": [
36+
"loader = TextLoader(r\"E:\\PythonInOffice\\langchain_egs\\00_getting_started\\sd+midjourney_wiki.txt\")\n",
37+
"docs = loader.load()"
38+
]
39+
},
40+
{
41+
"cell_type": "code",
42+
"execution_count": null,
43+
"id": "dc4dc1b4-2893-4a7d-81a6-8062d6934d17",
44+
"metadata": {},
45+
"outputs": [],
46+
"source": [
47+
"docs\n",
48+
"len(docs[0].page_content)"
49+
]
50+
},
51+
{
52+
"cell_type": "code",
53+
"execution_count": null,
54+
"id": "97744adb-8539-4907-a5a8-330933c2896c",
55+
"metadata": {},
56+
"outputs": [],
57+
"source": [
58+
"import tiktoken\n",
59+
"encoding = tiktoken.encoding_for_model(\"gpt-3.5-turbo\")"
60+
]
61+
},
62+
{
63+
"cell_type": "code",
64+
"execution_count": null,
65+
"id": "566fb240-7cd1-49a0-9ca5-a7b692e773c4",
66+
"metadata": {},
67+
"outputs": [],
68+
"source": [
69+
"encoding.encode(docs[0].page_content)\n",
70+
"len(encoding.encode(docs[0].page_content))"
71+
]
72+
},
73+
{
74+
"cell_type": "code",
75+
"execution_count": null,
76+
"id": "db02e3fe-9d95-48c1-b17e-aefc984b3b4f",
77+
"metadata": {},
78+
"outputs": [],
79+
"source": []
80+
},
81+
{
82+
"cell_type": "code",
83+
"execution_count": null,
84+
"id": "2de94ed1-944c-4b44-91d6-67d82312c613",
85+
"metadata": {},
86+
"outputs": [],
87+
"source": [
88+
"text_splitter = RecursiveCharacterTextSplitter(chunk_size = 4000, chunk_overlap=0)\n",
89+
"docs = text_splitter.split_documents(docs)"
90+
]
91+
},
92+
{
93+
"cell_type": "code",
94+
"execution_count": null,
95+
"id": "3c1992fc-7bc3-4b35-b78d-8475b01f5ba1",
96+
"metadata": {},
97+
"outputs": [],
98+
"source": [
99+
"len(docs)"
100+
]
101+
},
102+
{
103+
"cell_type": "code",
104+
"execution_count": null,
105+
"id": "f9d4cb12-cf4d-4736-a165-d7d56668b31e",
106+
"metadata": {},
107+
"outputs": [],
108+
"source": [
109+
"len(docs[1].page_content)"
110+
]
111+
},
112+
{
113+
"cell_type": "code",
114+
"execution_count": null,
115+
"id": "300a8268-2d8e-4c7a-a6e3-eec97defda54",
116+
"metadata": {},
117+
"outputs": [],
118+
"source": [
119+
"docs[0]"
120+
]
121+
},
122+
{
123+
"cell_type": "code",
124+
"execution_count": null,
125+
"id": "49746411-61b2-49e5-90b7-462c2b23e63c",
126+
"metadata": {},
127+
"outputs": [],
128+
"source": []
129+
},
130+
{
131+
"cell_type": "code",
132+
"execution_count": null,
133+
"id": "4244e652-4ae3-4d9d-9b6f-e2a50723553b",
134+
"metadata": {},
135+
"outputs": [],
136+
"source": [
137+
"llm = OpenAI(temperature=0, model_name='gpt-3.5-turbo')\n",
138+
"stuff_chain = load_summarize_chain(llm, chain_type='stuff', verbose=True)\n",
139+
"\n",
140+
"#will show an error\n",
141+
"stuff_chain.run(docs[:2])"
142+
]
143+
},
144+
{
145+
"cell_type": "code",
146+
"execution_count": null,
147+
"id": "56aa54f6-4d99-4947-ab89-dd4bdaa007bd",
148+
"metadata": {},
149+
"outputs": [],
150+
"source": [
151+
"llm = OpenAI(temperature=0)\n",
152+
"\n",
153+
"map_reduce_chain = load_summarize_chain(llm, chain_type='map_reduce', verbose=True)\n",
154+
"\n",
155+
"map_reduce_chain.run(docs)"
156+
]
157+
},
158+
{
159+
"cell_type": "code",
160+
"execution_count": null,
161+
"id": "38b3a5c2-af26-4565-853e-d9bc4035b9f4",
162+
"metadata": {},
163+
"outputs": [],
164+
"source": [
165+
"refine_chain = load_summarize_chain(llm, chain_type='refine', verbose=True)\n",
166+
"refine_chain.run(docs)"
167+
]
168+
},
169+
{
170+
"cell_type": "code",
171+
"execution_count": null,
172+
"id": "7f5357cb-dccc-43e1-8776-a7b07cc071b6",
173+
"metadata": {},
174+
"outputs": [],
175+
"source": [
176+
"from langchain.chains.question_answering import load_qa_chain\n",
177+
"llm = OpenAI(temperature=0)\n",
178+
"map_rerank_chain = load_qa_chain(llm, chain_type='map_rerank', verbose=True, return_intermediate_steps=True)"
179+
]
180+
},
181+
{
182+
"cell_type": "code",
183+
"execution_count": null,
184+
"id": "bffc4d75-8403-49f5-b9b9-856a7de7d483",
185+
"metadata": {},
186+
"outputs": [],
187+
"source": [
188+
"map_rerank_chain({'input_documents': docs,\n",
189+
" 'question': \"what's the main feature of midjourney?\"})"
190+
]
191+
},
192+
{
193+
"cell_type": "code",
194+
"execution_count": null,
195+
"id": "525a1ada-a16f-4e2f-a423-6e36e50b6362",
196+
"metadata": {},
197+
"outputs": [],
198+
"source": [
199+
"import tiktoken\n",
200+
"import openai\n",
201+
"import os\n",
202+
"from dotenv import dotenv_values\n",
203+
"\n",
204+
"api_keys=dotenv_values()\n",
205+
"os.environ['OPENAI_API_KEY'] = dotenv_values()['openai_api_key'] #set environment variable"
206+
]
207+
},
208+
{
209+
"cell_type": "code",
210+
"execution_count": null,
211+
"id": "0e919642-d7e3-4756-9815-e68a16c473e0",
212+
"metadata": {},
213+
"outputs": [],
214+
"source": [
215+
"encoding = tiktoken.encoding_for_model(\"gpt-3.5-turbo\") # gpt-4, gpt-3.5-turbo, text-embedding-ada-002"
216+
]
217+
},
218+
{
219+
"cell_type": "code",
220+
"execution_count": null,
221+
"id": "1e906311-fc67-451e-bc21-f76b24d2065b",
222+
"metadata": {},
223+
"outputs": [],
224+
"source": [
225+
"encoding.encode(\"let's count tokens!\")"
226+
]
227+
},
228+
{
229+
"cell_type": "code",
230+
"execution_count": null,
231+
"id": "f34ac127-a76a-499a-9614-69dec59d0e65",
232+
"metadata": {},
233+
"outputs": [],
234+
"source": [
235+
"token_num = len(encoding.encode(\"let's learn about tokens!\"))\n",
236+
"print(f'number of token is: {token_num}')"
237+
]
238+
},
239+
{
240+
"cell_type": "code",
241+
"execution_count": null,
242+
"id": "00f2e831-238d-4ab0-bfd8-b5b22cee4442",
243+
"metadata": {},
244+
"outputs": [],
245+
"source": [
246+
"import tiktoken\n",
247+
"\n",
248+
"#encode\n",
249+
"encoding = tiktoken.encoding_for_model(\"gpt-3.5-turbo\") # gpt-4, gpt-3.5-turbo, text-embedding-ada-002\n",
250+
"encoding.encode(\"guess how many tokens are in this sentence?\")"
251+
]
252+
},
253+
{
254+
"cell_type": "code",
255+
"execution_count": null,
256+
"id": "8615c8f3-2db3-4117-8964-a7a93a94cd6c",
257+
"metadata": {},
258+
"outputs": [],
259+
"source": [
260+
"token_num = len(encoding.encode(\"guess how many tokens are in this sentence?\"))\n",
261+
"print(f'number of token is: {token_num}')"
262+
]
263+
},
264+
{
265+
"cell_type": "code",
266+
"execution_count": null,
267+
"id": "362fbdd0-9e51-4615-8ba8-118f48a18925",
268+
"metadata": {},
269+
"outputs": [],
270+
"source": [
271+
"## turn token into text\n",
272+
"encoding.decode([52851, 1268, 1690, 11460, 527, 304, 420, 11914, 30])"
273+
]
274+
}
275+
],
276+
"metadata": {
277+
"kernelspec": {
278+
"display_name": "langchain-venv",
279+
"language": "python",
280+
"name": "langchain-venv"
281+
},
282+
"language_info": {
283+
"codemirror_mode": {
284+
"name": "ipython",
285+
"version": 3
286+
},
287+
"file_extension": ".py",
288+
"mimetype": "text/x-python",
289+
"name": "python",
290+
"nbconvert_exporter": "python",
291+
"pygments_lexer": "ipython3",
292+
"version": "3.10.8"
293+
}
294+
},
295+
"nbformat": 4,
296+
"nbformat_minor": 5
297+
}

0 commit comments

Comments
 (0)