Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -2,6 +2,20 @@ import gradio as gr
|
|
| 2 |
import tiktoken
|
| 3 |
import random
|
| 4 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
# Load the tokenizers
|
| 6 |
enc_gpt4o = tiktoken.encoding_for_model("gpt-4o")
|
| 7 |
enc_gpt3_5turbo = tiktoken.encoding_for_model("gpt-3.5-turbo")
|
|
@@ -37,7 +51,7 @@ def tokenize_input(text):
|
|
| 37 |
num_chars_html = f'<h2>Number of Characters: <span style="font-size: 20px; font-weight: bold;">{num_chars}</span></h2>'
|
| 38 |
return num_chars_html, gpt4o_result, gpt35turbo_result
|
| 39 |
|
| 40 |
-
|
| 41 |
with gr.Blocks() as demo:
|
| 42 |
gr.Markdown("## GPT Tokenizer Comparison App")
|
| 43 |
with gr.Row():
|
|
@@ -51,4 +65,6 @@ with gr.Blocks() as demo:
|
|
| 51 |
input_text.submit(tokenize_input, inputs=[input_text], outputs=[num_chars_output, gpt4o_output, gpt35turbo_output])
|
| 52 |
|
| 53 |
# Launch the app
|
| 54 |
-
demo.launch()
|
|
|
|
|
|
|
|
|
| 2 |
import tiktoken
|
| 3 |
import random
|
| 4 |
|
| 5 |
+
# License Information
|
| 6 |
+
# This application uses the following open-source libraries:
|
| 7 |
+
#
|
| 8 |
+
# 1. Gradio:
|
| 9 |
+
# - License: Apache License 2.0
|
| 10 |
+
# - Copyright: 2020-2023, Gradio contributors
|
| 11 |
+
# - Full License: http://www.apache.org/licenses/LICENSE-2.0
|
| 12 |
+
#
|
| 13 |
+
# 2. tiktoken:
|
| 14 |
+
# - License: MIT License
|
| 15 |
+
# - Copyright: 2022, OpenAI, Shantanu Jain
|
| 16 |
+
# - Full License: https://opensource.org/licenses/MIT
|
| 17 |
+
|
| 18 |
+
|
| 19 |
# Load the tokenizers
|
| 20 |
enc_gpt4o = tiktoken.encoding_for_model("gpt-4o")
|
| 21 |
enc_gpt3_5turbo = tiktoken.encoding_for_model("gpt-3.5-turbo")
|
|
|
|
| 51 |
num_chars_html = f'<h2>Number of Characters: <span style="font-size: 20px; font-weight: bold;">{num_chars}</span></h2>'
|
| 52 |
return num_chars_html, gpt4o_result, gpt35turbo_result
|
| 53 |
|
| 54 |
+
|
| 55 |
with gr.Blocks() as demo:
|
| 56 |
gr.Markdown("## GPT Tokenizer Comparison App")
|
| 57 |
with gr.Row():
|
|
|
|
| 65 |
input_text.submit(tokenize_input, inputs=[input_text], outputs=[num_chars_output, gpt4o_output, gpt35turbo_output])
|
| 66 |
|
| 67 |
# Launch the app
|
| 68 |
+
demo.launch()
|
| 69 |
+
|
| 70 |
+
|