你如何将文本传递给 tensorflow 模型以返回预测

tf/python 的新手,并创建了一个模型,该模型根据毒性级别(淫秽、有毒、威胁等)对文本进行分类。到目前为止,这是我所拥有的,它确实生成了摘要,所以我知道它正在正确加载。如何将文本传递给模型以返回预测?任何帮助将不胜感激。

import os
import numpy as np
import tensorflow as tf
from tensorflow import keras

checkpoint_path = "tf_model/the_model/saved_model.pb"
checkpoint_dir = os.path.dirname(checkpoint_path)

new_model = tf.keras.models.load_model(checkpoint_dir)

# Check its architecture
new_model.summary()

inputs = [
    "tenserflow seems like it fits the bill but there are zero tutorials that outline how to reuse a model in a production environment "
]

predictions = new_model.predict(inputs)
print(predictions)

我收到很多错误信息,其中一些冗长的错误信息如下:

警告:tensorflow:模型是用形状 (None,150) 构建的,用于输入 KerasTensor(type_spec=TensorSpec(shape=(None,150),dtype=tf.float32,name='input_1'),name='input_1 ',description="created by layer 'input_1'"),但它是在形状不兼容的输入上调用的(无,1)。

ValueError: 由 1 减去 3 for '{{node model/conv1d/conv1d}} = Conv2D[T=DT_FLOAT,data_format="NHWC",dilations=[1,1,1] 引起的负维度大小,explicit_paddings=[],padding="VALID",strides=[1,1],use_cudnn_on_gpu=true](model/conv1d/conv1d/ExpandDims,model/conv1d/conv1d/ExpandDims_1)' 输入形状: [?,256],[1,3,256,64]。

这是用于创建和测试它/预测完美的py代码:

import tensorflow as tf
import numpy as np
import pandas as pd

import os

TRAIN_DATA = "datasets/train.csv"
GLOVE_EMBEDDING = "embedding/glove.6B.100d.txt"

train = pd.read_csv(TRAIN_DATA)

train["comment_text"].fillna("fillna")

x_train = train["comment_text"].str.lower()
y_train = train[["toxic","severe_toxic","obscene","threat","insult","identity_hate"]].values

max_words = 100000
max_len = 150

embed_size = 100

tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=max_words,lower=True)

tokenizer.fit_on_texts(x_train)

x_train = tokenizer.texts_to_sequences(x_train)

x_train = tf.keras.preprocessing.sequence.pad_sequences(x_train,maxlen=max_len)

embeddings_index = {}

with open(GLOVE_EMBEDDING,encoding='utf8') as f:
    for line in f:
        values = line.rstrip().rsplit(' ')
        word = values[0]
        embed = np.asarray(values[1:],dtype='float32')
        embeddings_index[word] = embed

word_index = tokenizer.word_index

num_words = min(max_words,len(word_index) + 1)

embedding_matrix = np.zeros((num_words,embed_size),dtype='float32')

for word,i in word_index.items():

    if i >= max_words:
        continue

    embedding_vector = embeddings_index.get(word)

    if embedding_vector is not None:
        embedding_matrix[i] = embedding_vector

input = tf.keras.layers.Input(shape=(max_len,))

x = tf.keras.layers.Embedding(max_words,embed_size,weights=[embedding_matrix],trainable=False)(input)

x = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(128,return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(x)

x = tf.keras.layers.Conv1D(64,kernel_size=3,padding="valid",kernel_initializer="glorot_uniform")(x)

avg_pool = tf.keras.layers.GlobalAveragePooling1D()(x)
max_pool = tf.keras.layers.GlobalMaxPooling1D()(x)

x = tf.keras.layers.concatenate([avg_pool,max_pool])

preds = tf.keras.layers.Dense(6,activation="sigmoid")(x)

model = tf.keras.Model(input,preds)

model.summary()

model.compile(loss='binary_crossentropy',optimizer=tf.keras.optimizers.Adam(lr=1e-3),metrics=['accuracy'])

batch_size = 128

checkpoint_path = "tf_model/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)

cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path,save_weights_only=True,verbose=1)

callbacks = [
    tf.keras.callbacks.EarlyStopping(patience=5,monitor='val_loss'),tf.keras.callbacks.TensorBoard(log_dir='logs'),cp_callback
]

model.fit(x_train,y_train,validation_split=0.2,batch_size=batch_size,epochs=1,callbacks=callbacks,verbose=1)

latest = tf.train.latest_checkpoint(checkpoint_dir)

model.load_weights(latest)

# Save the entire model as a Savedmodel.
model.save('tf_model/the_model')

predictions = model.predict(np.expand_dims(x_train[42],0))
print(tokenizer.sequences_to_texts([x_train[42]]))
print(y_train[42])
print(predictions)

最终解决方案:

import os
import numpy as np
import tensorflow as tf
from tensorflow import keras

checkpoint_path = "tf_model/the_model/saved_model.pb"
checkpoint_dir = os.path.dirname(checkpoint_path)
new_model = tf.keras.models.load_model(checkpoint_dir)

max_words = 100000
max_len = 150

# Check its architecture
# new_model.summary()

inputs = ["tenserflow seems like it fits the bill but there are zero tutorials that outline how to reuse a model in a production environment."]

# use same tokenizer used to build model
tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=max_words,lower=True)
tokenizer.fit_on_texts(inputs)

# pass string to tokenizer and that 'array' is passed to predict
sequence = tokenizer.texts_to_sequences(inputs) # same tokenizer which is used on train data.
sequence = tf.keras.preprocessing.sequence.pad_sequences(sequence,maxlen = max_len)
predictions = new_model.predict(sequence)
print(predictions)
# [[0.0365479  0.01275077 0.02102855 0.00647011 0.02302513 0.00406089]]
haishizhende 回答:你如何将文本传递给 tensorflow 模型以返回预测

同样需要处理。这可以通过以下方式完成:

inputs = [
"tenserflow seems like it fits the bill but there are zero tutorials that outline 
 how to reuse a model in a production environment"]

sequence = tokenizer.texts_to_sequences(inputs) # same tokenizer which is used on train data.
sequence = pad_sequences(sequence,maxlen = max_len)

predictions = new_model.predict(sequence)
本文链接:https://www.f2er.com/752.html

大家都在问