Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def main(max_epoch):
_ = init_nncontext()
(training_images_data, training_labels_data) = mnist.read_data_sets("/tmp/mnist", "train")
(testing_images_data, testing_labels_data) = mnist.read_data_sets("/tmp/mnist", "test")
training_images_data = (training_images_data - mnist.TRAIN_MEAN) / mnist.TRAIN_STD
testing_images_data = (testing_images_data - mnist.TRAIN_MEAN) / mnist.TRAIN_STD
model = tf.keras.Sequential(
[tf.keras.layers.Flatten(input_shape=(28, 28, 1)),
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(10, activation='softmax'),
]
)
model.compile(optimizer='rmsprop',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
keras_model = KerasModel(model)
keras_model.fit(training_images_data,
def get_mnist(sc, data_type="train", location="/tmp/mnist"):
"""
Get and normalize the mnist data. We would download it automatically
if the data doesn't present at the specific location.
:param sc: SparkContext
:param data_type: training data or testing data
:param location: Location storing the mnist
:return: A RDD of (features: Ndarray, label: Ndarray)
"""
(images, labels) = mnist.read_data_sets(location, data_type)
images = sc.parallelize(images)
labels = sc.parallelize(labels + 1) # Target start from 1 in BigDL
record = images.zip(labels)
return record
def main(max_epoch):
_ = init_nncontext()
(training_images_data, training_labels_data) = mnist.read_data_sets("/tmp/mnist", "train")
(testing_images_data, testing_labels_data) = mnist.read_data_sets("/tmp/mnist", "test")
training_images_data = (training_images_data - mnist.TRAIN_MEAN) / mnist.TRAIN_STD
testing_images_data = (testing_images_data - mnist.TRAIN_MEAN) / mnist.TRAIN_STD
model = tf.keras.Sequential(
[tf.keras.layers.Flatten(input_shape=(28, 28, 1)),
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(10, activation='softmax'),
]
)
model.compile(optimizer='rmsprop',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
.map(lambda rec_tuple: ((rec_tuple[0] - mnist.TRAIN_MEAN) / mnist.TRAIN_STD,
np.array(rec_tuple[1])))
return rdd
def get_data_rdd(dataset, sc):
from bigdl.dataset import mnist
(images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", dataset)
image_rdd = sc.parallelize(images_data)
labels_rdd = sc.parallelize(labels_data)
rdd = image_rdd.zip(labels_rdd) \
.map(lambda rec_tuple: ((rec_tuple[0] - mnist.TRAIN_MEAN) / mnist.TRAIN_STD,
np.array(rec_tuple[1])))
return rdd
record = images.zip(labels).map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD),
rec_tuple[1])) \
.map(lambda t: Sample.from_ndarray(t[0], t[1]))
.map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TEST_MEAN, mnist.TEST_STD),
rec_tuple[1]))\
.map(lambda t: Sample.from_ndarray(t[0], t[1]))
.map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD),
rec_tuple[1]))\
.map(lambda t: Sample.from_ndarray(t[0], t[1]))
.map(lambda rec_tuple: [normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD),
np.array(rec_tuple[1])])
def get_mnist(sc, data_type="train", location="/tmp/mnist"):
"""
Download or load MNIST dataset to/from the specified path.
Normalize and transform input data into an RDD of Sample
"""
from bigdl.dataset import mnist
from bigdl.dataset.transformer import normalizer
(images, labels) = mnist.read_data_sets(location, data_type)
images = images.reshape((images.shape[0], ) + input_shape)
images = sc.parallelize(images)
labels = sc.parallelize(labels + 1) # Target start from 1 in BigDL
record = images.zip(labels).map(lambda rec_tuple: (normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD),
rec_tuple[1])) \
.map(lambda t: Sample.from_ndarray(t[0], t[1]))
return record