Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
hash_fields = {k: fields[k] for k in fields if k not in FIELDS_NOT_HASH and SEPRTR not in k}
h = hash(tuple(sorted(hash_fields.items())))
# Now add the hashed component to the group with the matching hash
# or create a new group if the hash hasn't been seen before.
try:
# Add next ref for identical part to the list.
component_groups[h].refs.append(ref)
# Also add any manufacturer's part number (or None) and each distributor
# stock catologue code to the group's list.
for f in FIELDS_MANFCAT:
component_groups[h].manfcat_codes[f].add(fields.get(f))
except KeyError:
# This happens if it is the first part in a group, so the group
# doesn't exist yet.
component_groups[h] = IdenticalComponents() # Add empty structure.
component_groups[h].refs = [ref] # Init list of refs with first ref.
# Now add the manf. part code (or None) and each distributor stock
# catologue code for this part to the group set.
component_groups[h].manfcat_codes = {}
for f in FIELDS_MANFCAT:
component_groups[h].manfcat_codes[f] = set([fields.get(f)])
#print('\n\n\n1++++++++++++++',len(component_groups))
#for g,grp in list(component_groups.items()):
# print('\n', grp.refs)
# for r in grp.refs:
# print(r, components[r])
# Now we have groups of seemingly identical parts. But some of the parts
# within a group may have different manufacturer's part numbers, and these
# groups may need to be split into smaller groups of parts all having the
# One manf# or cat# that is `None`. Don't split this
# group. These parts are not intended to be purchased.
# CASE FOUR:
# Otherwise, split the group into subgroups, each with the
# same manf# and distributors catalogue codes (for that one
# that will be scraped, the other ones are not considered).
for i_manfcat in range(max([len(grp.manfcat_codes.get(f)) for f in FIELDS_MANFCAT])):
manfcat_num = {}
for f in FIELDS_MANFCAT:
try:
manfcat_num[f] = list(grp.manfcat_codes.get(f))[i_manfcat]
except IndexError:
# If not have more code in the set list, is because just
# exist one. So use this as general.
manfcat_num[f] = list(grp.manfcat_codes.get(f))[0]
sub_group = IdenticalComponents()
sub_group.manfcat_codes = [manfcat_num]
sub_group.refs = []
for ref in grp.refs:
# Use get() which returns `None` if the component has no
# manf# or distributor# field. That will match if the
# group manf_num is also None. So append the par to the group.
if all([components[ref].get(f)==manfcat_num[f] for f in FIELDS_MANFCAT]):
sub_group.refs.append(ref)
new_component_groups.append(sub_group) # Append one part of the split group.
#print('\n\n\n2++++++++++++++',len(new_component_groups))
#for grp in new_component_groups:
# print('\n', grp.refs)
# for r in grp.refs:
# print(r, components[r])
# If the identical components grouped have difference in the `fields_merge`
# One manf# or cat# that is `None`. Don't split this
# group. These parts are not intended to be purchased.
# CASE FOUR:
# Otherwise, split the group into subgroups, each with the
# same manf# and distributors catalogue codes (for that one
# that will be scraped, the other ones are not considered).
for i_manfcat in range(max([len(grp.manfcat_codes.get(f)) for f in FIELDS_MANFCAT])):
manfcat_num = {}
for f in FIELDS_MANFCAT:
try:
manfcat_num[f] = list(grp.manfcat_codes.get(f))[i_manfcat]
except IndexError:
# If not have more code in the set list, is because just
# exist one. So use this as general.
manfcat_num[f] = list(grp.manfcat_codes.get(f))[0]
sub_group = IdenticalComponents()
sub_group.manfcat_codes = [manfcat_num]
sub_group.refs = []
for ref in grp.refs:
# Use get() which returns `None` if the component has no
# manf# or distributor# field. That will match if the
# group manf_num is also None. So append the par to the group.
if all([components[ref].get(f)==manfcat_num[f] for f in FIELDS_MANFCAT]):
sub_group.refs.append(ref)
new_component_groups.append(sub_group) # Append one part of the split group.
#print('\n\n\n2++++++++++++++',len(new_component_groups))
#for grp in new_component_groups:
# print('\n', grp.refs)
# for r in grp.refs:
# print(r, components[r])
# If the identical components grouped have difference in the `fields_merge`
hash_fields = {k: fields[k] for k in fields if k not in FIELDS_NOT_HASH and SEPRTR not in k}
h = hash(tuple(sorted(hash_fields.items())))
# Now add the hashed component to the group with the matching hash
# or create a new group if the hash hasn't been seen before.
try:
# Add next ref for identical part to the list.
component_groups[h].refs.append(ref)
# Also add any manufacturer's part number (or None) and each distributor
# stock catalogue code to the group's list.
for f in FIELDS_MANFCAT:
component_groups[h].manfcat_codes[f].add(fields.get(f))
except KeyError:
# This happens if it is the first part in a group, so the group
# doesn't exist yet.
component_groups[h] = IdenticalComponents() # Add empty structure.
component_groups[h].refs = [ref] # Init list of refs with first ref.
# Now add the manf. part code (or None) and each distributor stock
# catalogue code for this part to the group set.
component_groups[h].manfcat_codes = {}
for f in FIELDS_MANFCAT:
component_groups[h].manfcat_codes[f] = set([fields.get(f)])
#print('\n\n\n1++++++++++++++',len(component_groups))
#for g,grp in list(component_groups.items()):
# print('\n', grp.refs)
# for r in grp.refs:
# print(r, components[r])
# Now we have groups of seemingly identical parts. But some of the parts
# within a group may have different manufacturer's part numbers, and these
# groups may need to be split into smaller groups of parts all having the