Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
three_quarter = points_number * 3 // 4
# fills a fourth of data, from 2/4th to 3/4th
fill_archives(self.db, dst_db, time.time()-quarter, time.time()-half)
quarter_filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
expected = [None]*half + complete[half:three_quarter] + [None]*quarter
self.assertEqual(expected, quarter_filled_data)
# Fetching data older than start forces the use of the second level of aggregation
# We get a first empty cell and then
quarter_filled_data_l2 = whisper.fetch(dst_db, 0)[1]
average_l1 = _average(quarter_filled_data)
average_l2 = _average(quarter_filled_data_l2)
self.assertEqual(average_l1, average_l2)
# fills a half of data, from 2/4th to 4/4th
fill_archives(self.db, dst_db, time.time(), time.time()-half)
half_filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
expected = [None]*half + complete[half:]
self.assertEqual(expected, half_filled_data)
# Explicitly passes the default value of endAt=now (excluded)
fill_archives(self.db, dst_db, time.time(), endAt=0)
filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
self.assertEqual(complete[:-1], filled_data[:-1])
self.assertEqual(filled_data[-1], None)
empty_data = []
end = int(time.time()) + seconds_per_point
start = end - (points_number * seconds_per_point)
times = range(start, end, seconds_per_point)
complete_data = zip(times, complete)
self._createdb(self.db, schema, complete_data)
self._createdb(dst_db, schema, empty_data)
quarter = points_number // 4
half = points_number // 2
three_quarter = points_number * 3 // 4
# fills a fourth of data, from 2/4th to 3/4th
fill_archives(self.db, dst_db, time.time()-quarter, time.time()-half)
quarter_filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
expected = [None]*half + complete[half:three_quarter] + [None]*quarter
self.assertEqual(expected, quarter_filled_data)
# Fetching data older than start forces the use of the second level of aggregation
# We get a first empty cell and then
quarter_filled_data_l2 = whisper.fetch(dst_db, 0)[1]
average_l1 = _average(quarter_filled_data)
average_l2 = _average(quarter_filled_data_l2)
self.assertEqual(average_l1, average_l2)
# fills a half of data, from 2/4th to 4/4th
fill_archives(self.db, dst_db, time.time(), time.time()-half)
half_filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
expected = [None]*half + complete[half:]
self.assertEqual(expected, half_filled_data)
schema = [(1, 20)]
data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
end = int(time.time()) + schema[0][0]
start = end - (schema[0][1] * schema[0][0])
times = range(start, end, schema[0][0])
override_data = zip(times, data)
startTime = time.time()
self._createdb(self.db, schema)
self._createdb(testdb, schema, override_data)
fill_archives(self.db, testdb, startTime)
filled_data = whisper.fetch(testdb, 0)
self.assertEqual(data, filled_data[1])
def test_fill_empty(self):
testdb = "test-%s" % self.db
self._removedb()
try:
os.unlink(testdb)
except (IOError, OSError):
pass
schema = [(1, 20)]
emptyData = []
startTime = time.time()
self._createdb(self.db, schema)
self._createdb(testdb, schema, emptyData)
fill_archives(self.db, testdb, startTime)
original_data = whisper.fetch(self.db, 0)
filled_data = whisper.fetch(testdb, 0)
self.assertEqual(original_data, filled_data)
self.assertEqual(expected, quarter_filled_data)
# Fetching data older than start forces the use of the second level of aggregation
# We get a first empty cell and then
quarter_filled_data_l2 = whisper.fetch(dst_db, 0)[1]
average_l1 = _average(quarter_filled_data)
average_l2 = _average(quarter_filled_data_l2)
self.assertEqual(average_l1, average_l2)
# fills a half of data, from 2/4th to 4/4th
fill_archives(self.db, dst_db, time.time(), time.time()-half)
half_filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
expected = [None]*half + complete[half:]
self.assertEqual(expected, half_filled_data)
# Explicitly passes the default value of endAt=now (excluded)
fill_archives(self.db, dst_db, time.time(), endAt=0)
filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
self.assertEqual(complete[:-1], filled_data[:-1])
self.assertEqual(filled_data[-1], None)
6,
None, None, None, None,
11, 12, 13, 14, 15, 16, 17, 18, 19,
None, # trailing missing data point
]
end = int(time.time()) + schema[0][0]
start = end - (schema[0][1] * schema[0][0])
times = range(start, end, schema[0][0])
original_data = [t for t in zip(times, original) if t[1] is not None]
holes_data = [t for t in zip(times, holes) if t[1] is not None]
self._createdb(self.db, schema, original_data)
self._createdb(testdb, schema, holes_data)
fill_archives(self.db, testdb, time.time())
filled_data = whisper.fetch(testdb, 0)
self.assertEqual(complete, filled_data[1])
help='Whisper destination file')
args = parser.parse_args()
src = args.source
dst = args.dest
if not os.path.isfile(src):
raise SystemExit('Source file not found.')
if not os.path.isfile(dst):
raise SystemExit('Destination file not found.')
startFrom = time()
fill_archives(src, dst, startFrom)
help='Whisper destination file')
args = parser.parse_args()
src = args.source
dst = args.dest
if not os.path.isfile(src):
raise SystemExit('Source file not found.')
if not os.path.isfile(dst):
raise SystemExit('Destination file not found.')
startFrom = time()
fill_archives(src, dst, startFrom)
def deleteMetric(metric,
storage_dir='/opt/graphite/storage/whisper',
trash_dir="/opt/graphite/storage/whisper/trash"):
oldMetric = metric_path(metric, storage_dir)
newMetric = metric_path(metric, trash_dir)
if os.path.isfile(oldMetric):
try:
os.makedirs(os.path.dirname(newMetric))
except os.error:
pass
startFrom = time()
try:
fill_archives(oldMetric, newMetric, startFrom)
os.unlink(oldMetric)
except OSError, e:
if e.errno != errno.ENOENT:
raise
def heal_metric(source, dest):
try:
with open(dest):
try:
fill_archives(source, dest, time())
except CorruptWhisperFile as e:
logging.warn("Overwriting corrupt file %s!" % dest)
try:
os.makedirs(os.path.dirname(dest))
except os.error:
pass
try:
shutil.copyfile(source, dest)
except IOError as e:
logging.warn("Failed to copy %s! %s" % (dest, e))
except IOError:
try:
os.makedirs(os.path.dirname(dest))
except os.error:
pass
try: