Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def printResults(res):
printNumericTable(res.get(low_order_moments.minimum), "Minimum:")
printNumericTable(res.get(low_order_moments.maximum), "Maximum:")
printNumericTable(res.get(low_order_moments.sum), "Sum:")
printNumericTable(res.get(low_order_moments.sumSquares), "Sum of squares:")
printNumericTable(res.get(low_order_moments.sumSquaresCentered), "Sum of squared difference from the means:")
printNumericTable(res.get(low_order_moments.mean), "Mean:")
printNumericTable(res.get(low_order_moments.secondOrderRawMoment), "Second order raw moment:")
printNumericTable(res.get(low_order_moments.variance), "Variance:")
printNumericTable(res.get(low_order_moments.standardDeviation), "Standard deviation:")
printNumericTable(res.get(low_order_moments.variation), "Variation:")
# Read from the distributed HDFS data set at a specified path
dd = DistributedHDFSDataSet("/Spark/LowOrderMomentsCSR/data/")
dataRDD = dd.getCSRAsPairRDD(sc)
# Compute low order moments for dataRDD
result = runMoments(dataRDD)
# Print the results
minimum = result.get(low_order_moments.minimum)
maximum = result.get(low_order_moments.maximum)
sum = result.get(low_order_moments.sum)
sumSquares = result.get(low_order_moments.sumSquares)
sumSquaresCentered = result.get(low_order_moments.sumSquaresCentered)
mean = result.get(low_order_moments.mean)
secondOrderRawMoment = result.get(low_order_moments.secondOrderRawMoment)
variance = result.get(low_order_moments.variance)
standardDeviation = result.get(low_order_moments.standardDeviation)
variation = result.get(low_order_moments.variation)
# Redirect stdout to a file for correctness verification
stdout = sys.stdout
sys.stdout = open('LowOrderMomentsCSR.out', 'w')
print("Low order moments:")
printNumericTable(minimum, "Min:")
printNumericTable(maximum, "Max:")
printNumericTable(sum, "Sum:")
printNumericTable(sumSquares, "SumSquares:")
printNumericTable(sumSquaresCentered, "SumSquaredDiffFromMean:")
printNumericTable(mean, "Mean:")
printNumericTable(secondOrderRawMoment, "SecondOrderRawMoment:")
printNumericTable(variance, "Variance:")
if __name__ == "__main__":
# Create SparkContext that loads defaults from the system properties and the classpath and sets the name
sc = SparkContext(conf=SparkConf().setAppName("Spark low_order_moments(dense)").setMaster('local[4]'))
# Read from the distributed HDFS data set at a specified path
dd = DistributedHDFSDataSet("/Spark/LowOrderMomentsDense/data/")
dataRDD = dd.getAsPairRDD(sc)
# Compute low order moments for dataRDD
res = runMoments(dataRDD)
# Print the results
minimum = res.get(low_order_moments.minimum)
maximum = res.get(low_order_moments.maximum)
sum = res.get(low_order_moments.sum)
sumSquares = res.get(low_order_moments.sumSquares)
sumSquaresCentered = res.get(low_order_moments.sumSquaresCentered)
mean = res.get(low_order_moments.mean)
secondOrderRawMoment = res.get(low_order_moments.secondOrderRawMoment)
variance = res.get(low_order_moments.variance)
standardDeviation = res.get(low_order_moments.standardDeviation)
variation = res.get(low_order_moments.variation)
# Redirect stdout to a file for correctness verification
stdout = sys.stdout
sys.stdout = open('LowOrderMomentsDense.out', 'w')
print("Low order moments:")
printNumericTable(minimum, "Min:")
dd = DistributedHDFSDataSet("/Spark/LowOrderMomentsCSR/data/")
dataRDD = dd.getCSRAsPairRDD(sc)
# Compute low order moments for dataRDD
result = runMoments(dataRDD)
# Print the results
minimum = result.get(low_order_moments.minimum)
maximum = result.get(low_order_moments.maximum)
sum = result.get(low_order_moments.sum)
sumSquares = result.get(low_order_moments.sumSquares)
sumSquaresCentered = result.get(low_order_moments.sumSquaresCentered)
mean = result.get(low_order_moments.mean)
secondOrderRawMoment = result.get(low_order_moments.secondOrderRawMoment)
variance = result.get(low_order_moments.variance)
standardDeviation = result.get(low_order_moments.standardDeviation)
variation = result.get(low_order_moments.variation)
# Redirect stdout to a file for correctness verification
stdout = sys.stdout
sys.stdout = open('LowOrderMomentsCSR.out', 'w')
print("Low order moments:")
printNumericTable(minimum, "Min:")
printNumericTable(maximum, "Max:")
printNumericTable(sum, "Sum:")
printNumericTable(sumSquares, "SumSquares:")
printNumericTable(sumSquaresCentered, "SumSquaredDiffFromMean:")
printNumericTable(mean, "Mean:")
printNumericTable(secondOrderRawMoment, "SecondOrderRawMoment:")
printNumericTable(variance, "Variance:")
printNumericTable(standardDeviation, "StandartDeviation:")
if __name__ == "__main__":
comm = MPI.COMM_WORLD
rankId = comm.Get_rank()
# Initialize FileDataSource to retrieve the input data from a .csv file
dataSource = FileDataSource(datasetFileNames[rankId],
DataSourceIface.doAllocateNumericTable,
DataSourceIface.doDictionaryFromContext)
# Retrieve the input data
dataSource.loadDataBlock()
# Create an algorithm to compute low order moments on local nodes
localAlgorithm = low_order_moments.Distributed(step=step1Local)
# Set the input data set to the algorithm
localAlgorithm.input.set(low_order_moments.data, dataSource.getNumericTable())
# Compute low order moments
pres = localAlgorithm.compute()
# Serialize partial results required by step 2
dataArch = InputDataArchive()
pres.serialize(dataArch)
nodeResults = dataArch.getArchiveAsArray()
# Transfer partial results to step 2 on the root node
serializedData = comm.gather(nodeResults)
jp(DATA_PREFIX, 'covcormoments_csr_1.csv'),
jp(DATA_PREFIX, 'covcormoments_csr_2.csv'),
jp(DATA_PREFIX, 'covcormoments_csr_3.csv'),
jp(DATA_PREFIX, 'covcormoments_csr_4.csv')
]
if __name__ == "__main__":
comm = MPI.COMM_WORLD
rankId = comm.Get_rank()
# Retrieve the input data from a file
dataTable = createSparseTable(datasetFileNames[rankId])
# Create an algorithm to compute low order moments on local nodes
localAlgorithm = low_order_moments.Distributed(step1Local, method=low_order_moments.fastCSR)
# Set the input data set to the algorithm
localAlgorithm.input.set(low_order_moments.data, dataTable)
# Compute low order moments
pres = localAlgorithm.compute()
# Serialize partial results required by step 2
dataArch = InputDataArchive()
pres.serialize(dataArch)
nodeResults = dataArch.getArchiveAsArray()
# Transfer partial results to step 2 on the root node
serializedData = comm.gather(nodeResults)
def printResults(res):
printNumericTable(res.get(low_order_moments.minimum), "Minimum:")
printNumericTable(res.get(low_order_moments.maximum), "Maximum:")
printNumericTable(res.get(low_order_moments.sum), "Sum:")
printNumericTable(res.get(low_order_moments.sumSquares), "Sum of squares:")
printNumericTable(res.get(low_order_moments.sumSquaresCentered), "Sum of squared difference from the means:")
printNumericTable(res.get(low_order_moments.mean), "Mean:")
printNumericTable(res.get(low_order_moments.secondOrderRawMoment), "Second order raw moment:")
printNumericTable(res.get(low_order_moments.variance), "Variance:")
printNumericTable(res.get(low_order_moments.standardDeviation), "Standard deviation:")
printNumericTable(res.get(low_order_moments.variation), "Variation:")
def printResults(res):
printNumericTable(res.get(low_order_moments.minimum), "Minimum:")
printNumericTable(res.get(low_order_moments.maximum), "Maximum:")
printNumericTable(res.get(low_order_moments.sum), "Sum:")
printNumericTable(res.get(low_order_moments.sumSquares), "Sum of squares:")
printNumericTable(res.get(low_order_moments.sumSquaresCentered), "Sum of squared difference from the means:")
printNumericTable(res.get(low_order_moments.mean), "Mean:")
printNumericTable(res.get(low_order_moments.secondOrderRawMoment), "Second order raw moment:")
printNumericTable(res.get(low_order_moments.variance), "Variance:")
printNumericTable(res.get(low_order_moments.standardDeviation), "Standard deviation:")
printNumericTable(res.get(low_order_moments.variation), "Variation:")
return momentsMaster.finalizeCompute()
if __name__ == "__main__":
# Create SparkContext that loads defaults from the system properties and the classpath and sets the name
sc = SparkContext(conf=SparkConf().setAppName("Spark low_order_moments(sparse)").setMaster('local[4]'))
# Read from the distributed HDFS data set at a specified path
dd = DistributedHDFSDataSet("/Spark/LowOrderMomentsCSR/data/")
dataRDD = dd.getCSRAsPairRDD(sc)
# Compute low order moments for dataRDD
result = runMoments(dataRDD)
# Print the results
minimum = result.get(low_order_moments.minimum)
maximum = result.get(low_order_moments.maximum)
sum = result.get(low_order_moments.sum)
sumSquares = result.get(low_order_moments.sumSquares)
sumSquaresCentered = result.get(low_order_moments.sumSquaresCentered)
mean = result.get(low_order_moments.mean)
secondOrderRawMoment = result.get(low_order_moments.secondOrderRawMoment)
variance = result.get(low_order_moments.variance)
standardDeviation = result.get(low_order_moments.standardDeviation)
variation = result.get(low_order_moments.variation)
# Redirect stdout to a file for correctness verification
stdout = sys.stdout
sys.stdout = open('LowOrderMomentsCSR.out', 'w')
print("Low order moments:")
printNumericTable(minimum, "Min:")
def printResults(res):
printNumericTable(res.get(low_order_moments.minimum), "Minimum:")
printNumericTable(res.get(low_order_moments.maximum), "Maximum:")
printNumericTable(res.get(low_order_moments.sum), "Sum:")
printNumericTable(res.get(low_order_moments.sumSquares), "Sum of squares:")
printNumericTable(res.get(low_order_moments.sumSquaresCentered), "Sum of squared difference from the means:")
printNumericTable(res.get(low_order_moments.mean), "Mean:")
printNumericTable(res.get(low_order_moments.secondOrderRawMoment), "Second order raw moment:")
printNumericTable(res.get(low_order_moments.variance), "Variance:")
printNumericTable(res.get(low_order_moments.standardDeviation), "Standard deviation:")
printNumericTable(res.get(low_order_moments.variation), "Variation:")