Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
identified by sub_dir_path if provided in this temporary directory.
Return the location for this unique directory joined with the
sub_dir_path if any.
"""
# ensure that we have a new unique temp directory for each test run
global test_run_temp_dir
if not test_run_temp_dir:
from scancode_config import scancode_root_dir
test_tmp_root_dir = path.join(scancode_root_dir, 'tmp')
# now we add a space in the path for testing path with spaces
test_run_temp_dir = fileutils.get_temp_dir(
base_dir=test_tmp_root_dir, prefix='scancode-tk-tests -')
if on_linux and py2:
test_run_temp_dir = fsencode(test_run_temp_dir)
test_run_temp_subdir = fileutils.get_temp_dir(
base_dir=test_run_temp_dir, prefix='')
if sub_dir_path:
# create a sub directory hierarchy if requested
sub_dir_path = to_os_native_path(sub_dir_path)
test_run_temp_subdir = path.join(test_run_temp_subdir, sub_dir_path)
fileutils.create_dir(test_run_temp_subdir)
return test_run_temp_subdir
def get_temp_dir(self, sub_dir_path=None):
"""
Create a unique new temporary directory location. Create directories
identified by sub_dir_path if provided in this temporary directory.
Return the location for this unique directory joined with the
sub_dir_path if any.
"""
# ensure that we have a new unique temp directory for each test run
global test_run_temp_dir
if not test_run_temp_dir:
from scancode_config import scancode_root_dir
test_tmp_root_dir = path.join(scancode_root_dir, 'tmp')
# now we add a space in the path for testing path with spaces
test_run_temp_dir = fileutils.get_temp_dir(
base_dir=test_tmp_root_dir, prefix='scancode-tk-tests -')
if on_linux and py2:
test_run_temp_dir = fsencode(test_run_temp_dir)
test_run_temp_subdir = fileutils.get_temp_dir(
base_dir=test_run_temp_dir, prefix='')
if sub_dir_path:
# create a sub directory hierarchy if requested
sub_dir_path = to_os_native_path(sub_dir_path)
test_run_temp_subdir = path.join(test_run_temp_subdir, sub_dir_path)
fileutils.create_dir(test_run_temp_subdir)
return test_run_temp_subdir
def uncompress_file(location, decompressor):
"""
Uncompress a compressed file at location and return a temporary location of
the uncompressed file and a list of warning messages. Raise Exceptions on
errors. Use the `decompressor` object for decompression.
"""
# FIXME: do not create a sub-directory and instead strip the "compression"
# extension such gz, etc. or introspect the archive header to get the file
# name when present.
assert location
assert decompressor
warnings = []
base_name = fileutils.file_base_name(location)
target_location = os.path.join(fileutils.get_temp_dir(prefix='scancode-extract-'), base_name)
with decompressor(location, 'rb') as compressed:
with open(target_location, 'wb') as uncompressed:
buffer_size = 32 * 1024 * 1024
while True:
chunk = compressed.read(buffer_size)
if not chunk:
break
uncompressed.write(chunk)
if getattr(decompressor, 'has_trailing_garbage', False):
warnings.append(location + ': Trailing garbage found and ignored.')
return target_location, warnings
def get_cache_dir(base_cache_dir=scans_cache_dir):
"""
Return a new, created and unique cache storage directory.
"""
create_dir(base_cache_dir)
# create a unique temp directory in cache_dir
prefix = time2tstamp() + u'-'
cache_dir = get_temp_dir(base_cache_dir, prefix=prefix)
if on_linux:
cache_dir = path_to_bytes(cache_dir)
return cache_dir
Note: there are a few cases where the primary extractor for a type may fail and
a secondary extractor will succeed.
"""
abs_location = os.path.abspath(os.path.expanduser(location))
abs_target_dir = compat.unicode(os.path.abspath(os.path.expanduser(target_dir)))
# attempt extract first to a temp dir
temp_target1 = compat.unicode(fileutils.get_temp_dir(prefix='scancode-extract1-'))
try:
warnings = extractor1(abs_location, temp_target1)
if TRACE:
logger.debug('extract_with_fallback: temp_target1: %(temp_target1)r' % locals())
fileutils.copytree(temp_target1, abs_target_dir)
except:
try:
temp_target2 = compat.unicode(fileutils.get_temp_dir(prefix='scancode-extract2-'))
warnings = extractor2(abs_location, temp_target2)
if TRACE:
logger.debug('extract_with_fallback: temp_target2: %(temp_target2)r' % locals())
fileutils.copytree(temp_target2, abs_target_dir)
finally:
fileutils.delete(temp_target2)
finally:
fileutils.delete(temp_target1)
return warnings
def try_to_extract(location, target_dir, extractor):
"""
Extract archive at `location` to `target_dir` trying the `extractor` function.
If extract fails, just return without returning warnings nor raising exceptions.
Note: there are a few cases where we want to attempt extracting something
but do not care if this fails.
"""
abs_location = os.path.abspath(os.path.expanduser(location))
abs_target_dir = compat.unicode(os.path.abspath(os.path.expanduser(target_dir)))
temp_target = compat.unicode(fileutils.get_temp_dir(prefix='scancode-extract1-'))
warnings = []
try:
warnings = extractor(abs_location, temp_target)
if TRACE:
logger.debug('try_to_extract: temp_target: %(temp_target)r' % locals())
fileutils.copytree(temp_target, abs_target_dir)
except:
return warnings
finally:
fileutils.delete(temp_target)
return warnings
def get_database(database_file=None):
"""
Return the path to a new, temporary SQLite database file or an existing file if
database_file is provided.
"""
if not database_file:
import os
from commoncode import fileutils
tmp_dir = fileutils.get_temp_dir(base_dir='db')
database_file = os.path.join(tmp_dir, 'scancode.db')
return database_file
def convert_to_text(location, _retrying=False):
"""
Convert the markup file at location to plain text.
Return the location of the converted plain text file or None.
"""
if not is_markup(location):
return
temp_file = os.path.join(fileutils.get_temp_dir(prefix='scancode-markup-'), 'text')
from bs4 import BeautifulSoup
with open(location, 'rb') as input_text:
soup = BeautifulSoup(input_text.read(), 'html5lib')
with codecs.open(temp_file, mode='wb', encoding='utf-8') as output_text:
output_text.write(soup.get_text())
return temp_file