Update app.py

app.py

@@ -103,7 +103,7 @@ elif tokenizer == "Qwen2.5 Tokenizer":
     with st.expander("About Qwen2.5 Tokenizer"):
         st.write('')
     ids = qwen_tokenizer.encode(txt)
-    split_tokens = qwen_tokenizer.tokenize(txt)
+    split_tokens = [qwen_tokenizer.decode([t]) for t in ids]
     st.write(stream_data)
     if token_id == True:
         color = itertools.cycle(colors)
@@ -113,7 +113,7 @@ elif tokenizer == "RuAdapt Tokenizer":
     with st.expander("About RuAdapt Tokenizer"):
         st.write('')
     ids = ruadapt_tokenizer.encode(txt)
-    split_tokens = ruadapt_tokenizer.tokenize(txt)
+    split_tokens = [ruadapt_tokenizer.decode([t]) for t in ids]
     st.write(stream_data)
     if token_id == True:
         color = itertools.cycle(colors)