-
Notifications
You must be signed in to change notification settings - Fork 1
/
gpt_large_text_into_smaller_chunks.py
48 lines (35 loc) · 1.49 KB
/
gpt_large_text_into_smaller_chunks.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
from commands import OS, File, List, copy, paste
input_file = OS.args[1]
content = File.read(input_file)
output_file = input_file + ".smaller.txt"
CONTEXT_WINDOW = 4096
PROMPT_LENGTH_TOKENS = 200
TOKEN_MULTIPLIER_UNICODE = 1.5
TOKEN_MULTIPLIER_EN = 3.75
TOKEN_MULTIPLIER = TOKEN_MULTIPLIER_EN
if "u" in OS.args:
TOKEN_MULTIPLIER = TOKEN_MULTIPLIER_UNICODE
CHUNKS = List.split_every(content, int(CONTEXT_WINDOW*TOKEN_MULTIPLIER-PROMPT_LENGTH_TOKENS))
File.wipe(output_file)
total = len(CHUNKS)
for cnt, chunk in enumerate(CHUNKS):
chunk_cnt = cnt + 1
print(f"{chunk_cnt} / {total} chunk request copied to clipboard, feed it into chatgpt")
prompt = (f"Content: \"\"\"{chunk}\"\"\"\n"
+ f"It's chunk of data #{chunk_cnt} of {total} chunks total.\n\n"
+ "Please remove information that isn't valuable.\n\n"
+ "Only save the most valuable information that will be helpful.\n\n"
+ "Write all valuable information from this chunk of text.\n\n"
+ "Format it as code for easier copying, please.\n\n"
+ "Use markdown to emphasize information of the most importance. Thank you."
)
copy(prompt)
print("press Enter when answer is in clipboard")
input()
output = paste()
if prompt.strip() != output.strip():
print("saved new info")
File.write(output_file, f"<<<{chunk_cnt}>>>\n{output}\n\n", mode="a")
else:
print("no new info, skipping")
print(f"done, check file {output_file}")