change model and change for segmented data

app.py

@@ -10,6 +10,8 @@ from pathlib import Path
 import pandas as pd
 import plotly.express as px
 
+from underthesea import word_tokenize
+
 #Load tokenizer
 #fp = Path(__file__).with_name('tokenizer.pkl')
 with open('train/tokenizer/tokenizer.pkl',mode="rb") as f:
@@ -26,6 +28,7 @@ GRU_model = load_model('train/model/gru_model.h5')
 
 def tokenizer_pad(tokenizer,comment_text,max_length=200):
    
+    comment_text = word_tokenize(comment_text, format="text")
     comment_text = [comment_text]
     tokenized_text = tokenizer.texts_to_sequences(comment_text)
 

requirements.txt

@@ -5,4 +5,5 @@ tensorflow
 numpy
 pathlib
 plotly
-pandas
+pandas
+underthesea

train/model/gru_model.h5

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:96033429be73138b6a17600a9b88940e7f378dc56eeada17792788fe416699f7
-size 242665448
+oid sha256:d5ec54a3ff4cdd458ab651ca05774eb7f4f7e744046def14e9d176755a1a142a
+size 244995280

train/model/lstm_model.h5

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83f2080264aaf8caa00b382d9ddeeeb7cbc7c4b1a6d47c96b166d59f062a912a
-size 243514304
+oid sha256:f53c0db54bb35ec3649093abb0d0df3e40406003b16d83dc075e0c69127e8ed3
+size 243514400

train/tokenizer/tokenizer.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f88a7dfdd45db0f08ce565aac2fcacb559c46198bb5f587c51aeb60f9f2e479d
-size 4583690
+oid sha256:ddfc25d59703cbcb2d2267ff9a5ed142abf70bccd826b13b88952547308371ef
+size 9146630