Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def data_prep():
print 'Starting with data prep'
with open('../dataset/frame_count.pickle','rb') as f1:
frame_count=pickle.load(f1)
with open('../dataset/merged_data.pickle','rb') as f2:
merged_data=pickle.load(f2)
print 'Loaded dictionary'
root = './of_images'
path = os.path.join(root, '')
data={}
misplaced_data=[]
count=0
for path, subdirs, files in scandir.walk(root):
for filename in files:
print filename + ' ' + str(count)
count+=1
try:
vidname=filename.split('_',1)[1].split('.')[0]
fc=frame_count[vidname]
for i,j in enumerate(merged_data[vidname]):
if j:
index=i
break
for i in range(1,(fc/50)+1):
data[vidname+'@'+str(i)]=index+1
except:
misplaced_data.append(filename)
def normalized_walk(top, **kwargs):
"""
macOS uses decomposed UTF-8 to store filenames. This functions
is like `os.walk` but recomposes those decomposed filenames on
macOS
"""
try:
from scandir import walk
except:
# there is a warning in FileUtilities, so don't warn again here
from os import walk
if 'Darwin' == platform.system():
for dirpath, dirnames, filenames in walk(top, **kwargs):
yield dirpath, dirnames, [
unicodedata.normalize('NFC', to_unicode(fn)).encode(UTF8)
for fn in filenames
]
else:
if os.name == 'nt':
# NTFS stores files as Unicode, and this makes os.walk() return
# Unicode.
top2 = unicode(top)
else:
# On Linux the file system encoding may be UTF-8, but deal with
# bytestrings to avoid potential UnicodeDecodeError in
# posixpath.join()
top2 = str(top)
for result in walk(top2, **kwargs):
yield result
ignore_dirs (iterable): list of directories to ignore from search
maxdepth (int): maximum depth to recursively search
Returns:
list: list of files within location matching pattern
"""
results = []
if isinstance(ignore_dirs, str):
ignore_dirs = list(ignore_dirs)
location = os.path.normpath(location)
num_sep = location.count(os.path.sep) - 1
for root, dirs, files in walk(location, followlinks=True):
if ignore_dirs:
dirs[:] = [d for d in dirs if d not in ignore_dirs]
depth = root.count(os.path.sep) - num_sep
if depth > maxdepth:
dirs[:] = []
files[:] = []
for fname in fnmatch.filter(files, pattern):
results.append(os.path.abspath(os.path.join(root, fname)))
return results
**last_scanned_time** in the cached data.
2. Check the number of files and directories in the cached copy for any
differences.
"""
scan_directory = os.path.realpath(scan_directory).strip()
if self.directory_in_cache(scan_directory):
# Is in cache
st = os.stat(scan_directory)
# Return true if modified time on directory is newer Cached Time.
if self.d_cache[scan_directory].has_key("last_scanned_time"):
if st[stat.ST_MTIME] > self.d_cache[scan_directory]\
["last_scanned_time"]:
return True
path, raw_dirc, raw_filec = scandir.walk(scan_directory).next()
if (self.d_cache[scan_directory]["raw_filec"] != len(raw_filec) or
self.d_cache[scan_directory]["raw_dirc"] != len(raw_dirc)):
return True
return False
else:
# Does not exist in Cache, so force a load.
return True
def get_files(dir, notRecursive):
# Not Recursive
if notRecursive:
for filename in os.listdir(dir):
filePath = os.path.join(dir, filename)
if os.path.isdir(filePath):
continue
yield filePath
# Recursive
else:
for root, directories, files in scandir.walk(dir, followlinks=False):
for filename in files:
filePath = os.path.join(root, filename)
yield filePath
Returns:
(File_count, dir_count) - Tupple of Integers
This is a internal use only function.
It returns the number of files, and directories in the scan_directory.
This is the actual file system, and not cached directories.
* Primarily used to in directory_changed to help highlight directories
that have changed, but the last_modified has not been updated.
* Also used in _scan_directory_list, to help populate the directory
count of files / dirs.
"""
scan_directory = os.path.realpath(scan_directory).strip()
path, dirs, files = next(scandir.walk(scan_directory))
return (len(files), len(dirs))
hidden : bool
If True, include hidden files.
Returns
-------
View
A View with files in this Tree as members.
"""
from .collections import View
if not self.exists:
raise OSError("Tree doesn't exist in the filesystem")
out = []
for root, dirs, files in scandir.walk(self.abspath):
if hidden:
out = [Leaf(os.path.join(root, f)) for f in files]
else:
out = [Leaf(os.path.join(root, f)) for f in files
if f[0] != os.extsep]
break
out.sort()
return View(out)
def scanPath(path, rule_sets, filename_iocs, filename_suspicious_iocs, hashes, false_hashes):
# Startup
log("INFO","Scanning %s ... " % path)
# Counter
c = 0
# Get application path
appPath = getApplicationPath()
# Linux excludes from mtab
if isLinux:
allExcludes = LINUX_PATH_SKIPS_START | Set(getExcludedMountpoints())
for root, directories, files in scandir.walk(path, onerror=walkError, followlinks=False):
if isLinux:
# Skip paths that start with ..
newDirectories = []
for dir in directories:
skipIt = False
completePath = os.path.join(root, dir)
for skip in allExcludes:
if completePath.startswith(skip):
log("INFO", "Skipping %s directory" % skip)
skipIt = True
if not skipIt:
newDirectories.append(dir)
directories[:] = newDirectories
# Loop through files
Returns:
(File_count, dir_count) - Tupple of Integers
This is a internal use only function.
It returns the number of files, and directories in the scan_directory.
This is the actual file system, and not cached directories.
* Primarily used to in directory_changed to help highlight directories
that have changed, but the last_modified has not been updated.
* Also used in _scan_directory_list, to help populate the directory
count of files / dirs.
"""
scan_directory = os.path.realpath(scan_directory).strip()
path, dirs, files = scandir.walk(scan_directory).next()
return (len(files), len(dirs))
def get_file_names():
filenames = []
for folder, _, files in walk(icloudpath):
for filename in files:
filenames.append((folder + '/' + filename)[len(icloudpath) + 1:])
return filenames