我正在尝试使用keras和tensorflow 1.15在GCP上训练模型。 从现在开始,我的代码类似于我可以在colab上执行的代码:
it('should have font-weight 600',async () => {
const { getByText } = render(<Tabs tabItems={tabItems} selected="test1" />);
expect(getByText('Test 1').parentElement).toHaveStyleRule(
'font-weight','600',{ modifier: css`> button` }
);
});
但是我的数据在存储桶中,而我的代码在VM中。那我该怎么办?我尝试使用“ gs:// BUCKETS”加载数据,但是它不起作用。我该怎么办 ? 编辑:我添加我的代码以加载数据,我对不起。
# TPUs
import tensorflow as tf
print(tf.__version__)
cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver("tpu-name")
tf.config.experimental_connect_to_cluster(cluster_resolver)
tf.tpu.experimental.initialize_tpu_system(cluster_resolver)
tpu_strategy = tf.distribute.experimental.TPUStrategy(cluster_resolver)
print("Number of accelerators: ",tpu_strategy.num_replicas_in_sync)
import numpy as np
np.random.seed(123) # for reproducibility
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense,Dropout,activation,flatten
from tensorflow.keras.layers import Convolution2D,MaxPooling2D,Input
from tensorflow.keras import utils
from tensorflow.keras.datasets import mnist,cifar10
from tensorflow.keras.models import Model
# 4. Load data into train and test sets
(X_train,y_train) = load_data(sets="gs://BUCKETS/dogscats/train/",target_size=img_size)
(X_test,y_test) = load_data(sets="gs://BUCKETS/dogscats/valid/",target_size=img_size)
print(X_train.shape,X_test.shape)
# 5. Preprocess input data
#X_train = X_train.reshape(X_train.shape[0],28,1)
#X_test = X_test.reshape(X_test.shape[0],1)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255.0
X_test /= 255.0
print(y_train.shape,y_test.shape)
# 6. Preprocess class labels One hot encoding
Y_train = utils.to_categorical(y_train,2)
Y_test = utils.to_categorical(y_test,2)
print(Y_train.shape,Y_test.shape)
with tpu_strategy.scope():
model = make_model((img_size,img_size,3))
# 8. Compile model
model.compile(loss='categorical_crossentropy',optimizer="sgd",metrics=['accuracy'])
model.summary()
batch_size = 1250 * tpu_strategy.num_replicas_in_sync
# 9. Fit model on training data
model.fit(X_train,Y_train,steps_per_epoch=len(X_train)//batch_size,epochs=5,verbose=1)
EDIT2:如果其他人在同一情况下,请完成@daudnadeem的答案。
我的目标是从存储桶中获取图像,因此代码可以很好地工作并允许获取字节对象。要将其转换为图像,您只需要使用PIL库:
def load_data(sets="dogcats/train/",k = 5000,target_size=250):
# define location of dataset
folder = sets
photos,labels = list(),list()
# determine class
output = 0.0
for i,dog in enumerate(listdir(folder + "dogs/")):
if i >= k:
break
# load image
photo = load_img(folder + "dogs/" +dog,target_size=(target_size,target_size))
# convert to numpy array
photo = img_to_array(photo)
# store
photos.append(photo)
labels.append(output)
output = 1.0
for i,cat in enumerate(listdir(folder + "cats/") ):
if i >= k:
break
# load image
photo = load_img(folder + "cats/"+cat,target_size))
# convert to numpy array
photo = img_to_array(photo)
# store
photos.append(photo)
labels.append(output)
# convert to a numpy arrays
photos = asarray(photos)
labels = asarray(labels)
print(photos.shape,labels.shape)
photos,labels = shuffle(photos,labels,random_state=0)
return photos,labels