Hugging Face's logo

Spaces:
mponty
/
some_tokenization
Sleeping

App Files Community
mponty commited on
Commit
5d72c5f
·
verified ·
1 Parent(s): fd3c467

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +1 -1
app.py CHANGED
@@ -78,7 +78,7 @@ def stream_wp_token_ids():
78
 
79
  def num_tokens(txt):
80
  words = white_space_tokenizer(txt)
81
- n_words = len(n_words) if len(n_words) else 1
82
  try:
83
  return f'Token count {len(ids)}, f-rate {len(ids)/n_words}'
84
  except:
 
78
 
79
  def num_tokens(txt):
80
  words = white_space_tokenizer(txt)
81
+ n_words = len(words) if len(words) else 1
82
  try:
83
  return f'Token count {len(ids)}, f-rate {len(ids)/n_words}'
84
  except: