Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test():
base_dir = os.path.join(os.getcwd(), "data")
logging.info("base_dir: ".format(base_dir))
openurl = "url_ver=Z39.88-2004&rft_val_fmt=info:ofi/fmt:kev:mtx:journal&rft.atitle=Isolation of a common receptor for coxsackie B&rft.jtitle=Science&rft.aulast=Bergelson&rft.auinit=J&rft.date=1997&rft.volume=275&rft.spage=1320&rft.epage=1323"
metajson_list = crossref.query_openurl_and_retrieve_metadata(openurl, True)
if metajson_list:
output_path = os.path.join(base_dir, "result", "result_crossref_metajon.json")
logging.info(io_service.export_metajson_collection("test_crossref", "Crossref import test", metajson_list, output_path))
else:
assert False
def parse_and_convert_json(input_file_path, input_format, source, rec_id_prefix, only_first_record):
json_data = io_service.parse_json(input_file_path)
return convert_json(json_data, input_format, source, rec_id_prefix, only_first_record)
input_format = constants.FORMAT_UNIMARC
output_dir_path = os.path.join("data", "num", "output")
if not os.path.exists(output_dir_path):
os.mkdir(output_dir_path)
error_file_name = "".join(["validation-", corpus, ".txt"])
error_file_path = os.path.join(output_dir_path, error_file_name)
#logging.debug("error_file_path: {}".format(error_file_path))
# conf corpus
corpus_service.clean_corpus(corpus)
corpus_service.conf_corpus(corpus, "aime")
date_clean = datetime.datetime.now()
chrono.chrono_trace("Clean and conf corpus", date_begin, date_clean, None)
# import
input_file_paths = io_service.get_relevant_file_list_by_format(input_dir_path, input_format)
results = corpus_service.import_metadata_files(corpus, input_file_paths, input_format, source, rec_id_prefix, True, None)
date_import = datetime.datetime.now()
chrono.chrono_trace("Import corpus", date_clean, date_import, None)
# Validate
corpus_service.validate_corpus(corpus, error_file_path)
date_validate = datetime.datetime.now()
chrono.chrono_trace("Validate corpus", date_import, date_validate, None)
# Export mods
corpus_service.export_corpus(corpus, output_dir_path, constants.FORMAT_MODS, False, True)
date_export_mods = datetime.datetime.now()
chrono.chrono_trace("Export corpus mods", date_validate, date_export_mods, None)
# Export oai_dc
corpus_service.export_corpus(corpus, output_dir_path, constants.FORMAT_OAI_DC, False, True)
def parse_and_convert_bibtex(input_file_path, input_format, source, rec_id_prefix, only_first_record):
bibtex_root = io_service.parse_bibtex(input_file_path)
return convert_bibtext(bibtex_root, input_format, source, rec_id_prefix, only_first_record)
def parse_and_convert_file(input_file_path, input_format, output_format, source, rec_id_prefix, only_first_record, all_in_one_file):
""" Convert from a file path """
# input_format type determination
input_type = io_service.guess_type_from_format(input_format)
if input_type is None:
# file_extension type determination
input_type = io_service.guess_type_from_file(input_file_path)
#logging.info("parse_and_convert_file input_file_path: {}; input_format: {}; input_type: {}".format(input_file_path, input_format, input_type))
if input_type is not None:
metajson_list = None
if input_type == constants.FILE_TYPE_XMLETREE:
# xml
metajson_list = parse_and_convert_xmletree(input_file_path, input_format, source, rec_id_prefix, only_first_record)
elif input_type == constants.FILE_TYPE_JSON:
# json
metajson_list = parse_and_convert_json(input_file_path, input_format, source, rec_id_prefix, only_first_record)
def parse_and_convert_file(input_file_path, input_format, output_format, source, rec_id_prefix, only_first_record, all_in_one_file):
""" Convert from a file path """
# input_format type determination
input_type = io_service.guess_type_from_format(input_format)
if input_type is None:
# file_extension type determination
input_type = io_service.guess_type_from_file(input_file_path)
#logging.info("parse_and_convert_file input_file_path: {}; input_format: {}; input_type: {}".format(input_file_path, input_format, input_type))
if input_type is not None:
metajson_list = None
if input_type == constants.FILE_TYPE_XMLETREE:
# xml
metajson_list = parse_and_convert_xmletree(input_file_path, input_format, source, rec_id_prefix, only_first_record)
elif input_type == constants.FILE_TYPE_JSON:
# json
metajson_list = parse_and_convert_json(input_file_path, input_format, source, rec_id_prefix, only_first_record)
elif input_type == constants.FILE_TYPE_BIBTEX:
# bibtex
metajson_list = parse_and_convert_bibtex(input_file_path, input_format, source, rec_id_prefix, only_first_record)
def parse_and_convert_xmletree_str(input_string, input_format, source, rec_id_prefix, only_first_record):
xmletree_root = io_service.parse_xmletree_str(input_string)
return convert_xmletree(xmletree_root, input_format, source, rec_id_prefix, only_first_record)
def parse_and_convert_txt_lines(input_file_path, input_format, source, rec_id_prefix, only_first_record):
txt_lines = io_service.parse_txt_lines(input_file_path)
return convert_txt_lines(txt_lines, input_format, source, rec_id_prefix, only_first_record)
def convert_xmletree(xmletree_root, input_format, source, rec_id_prefix, only_first_record):
if xmletree_root is not None:
if input_format is None:
input_format = io_service.guess_format_from_xmletree(xmletree_root)
if input_format is not None:
logging.info("# input_format: {0}".format(input_format))
if input_format == constants.FORMAT_DDI:
# ddi
return ddi_crosswalk.ddi_xmletree_to_metajson_list(xmletree_root, source, rec_id_prefix, only_first_record)
elif input_format == constants.FORMAT_DIDL:
# didl
return didl_crosswalk.didl_xmletree_to_metajson_list(xmletree_root, source, rec_id_prefix, only_first_record)
elif input_format == constants.FORMAT_ENDNOTEXML:
# endnotexml
return endnotexml_crosswalk.endnotexml_xmletree_to_metajson_list(xmletree_root, source, rec_id_prefix, only_first_record)
metajson_list = repository_service.get_documents(corpus)
# one record per physical resource
if one_record_per_copy:
metajson_tmp = export_one_record_per_copy(metajson_list)
else:
metajson_tmp = metajson_list
# convert
results = crosswalks_service.convert_metajson_list(metajson_tmp, output_format, all_in_one_file)
# export
if all_in_one_file:
io_service.write_items_in_one_file(corpus, corpus, results, output_file_path, output_format)
else:
io_service.write_items(corpus, corpus, results, output_file_path, output_format)