How to use the sudachipy.dictionarylib.charactercategory.CharacterCategory.Range function in SudachiPy

To help you get started, we’ve selected a few SudachiPy examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github WorksApplications / SudachiPy / tests / dictionarylib / test_charactercategory.py View on Github external
def test_range_containing_length(self):
        range_ = charactercategory.CharacterCategory.Range()
        range_.low = 0x41
        range_.high = 0x54
        self.assertEqual(3, range_.containing_length('ABC12'))
        self.assertEqual(0, range_.containing_length('熙'))
github megagonlabs / ginza / sudachipy / dictionarylib / charactercategory.py View on Github external
range_.low = range_.high = int(r[0], 16)
            if len(r) > 1:
                range_.high = int(r[1], 16)
            if range_.low > range_.high:
                f.close()
                raise AttributeError("invalid range at line {}".format(i))
            for j in range(1, len(cols)):
                if re.match("#", cols[j]) or cols[j] is '':
                    break
                type_ = categorytype.CategoryType.get(cols[j])
                if type_ is None:
                    f.close()
                    raise AttributeError("{} is invalid type at line {}".format(cols[j], i))
                range_.categories.add(type_)
            self.range_list.append(range_)
        default_range = self.Range()
        default_range.low = 0
        default_range.high = float('inf')
        default_range.categories.add(categorytype.CategoryType.DEFAULT)
        self.range_list.reverse()
        self.range_list.append(default_range)

        f.close()
github WorksApplications / SudachiPy / sudachipy / dictionarylib / charactercategory.py View on Github external
left_chain.put(right)
                pivot = right.low
                states.extend(right.categories)
                continue
            left = left_chain.get()
            right = right_chain[0] if right_chain else None
            left_end = left.high
            right_begin = right.low if right else math.inf
            if left_end <= right_begin:
                new_range_list.append(self.Range(pivot, left_end, set(states)))
                pivot = left_end
                for cat in left.categories:
                    states.remove(cat)
                continue
            else:
                new_range_list.append(self.Range(pivot, right_begin, set(states)))
                pivot = right_begin
                states.extend(right.categories)
                left_chain.put(right)
                left_chain.put(left)
                right_chain.pop(0)
        self.range_list = []
        _range = new_range_list[0]
        for irange in new_range_list[1:]:
            if irange.low == _range.high and irange.categories == _range.categories:
                _range = self.Range(_range.low, irange.high, _range.categories)
            else:
                self.range_list.append(_range)
                _range = irange
        self.range_list.append(_range)
github WorksApplications / SudachiPy / sudachipy / dictionarylib / charactercategory.py View on Github external
pivot = 0
        while True:
            if left_chain.empty():
                if not right_chain:
                    break
                right = right_chain.pop(0)
                left_chain.put(right)
                pivot = right.low
                states.extend(right.categories)
                continue
            left = left_chain.get()
            right = right_chain[0] if right_chain else None
            left_end = left.high
            right_begin = right.low if right else math.inf
            if left_end <= right_begin:
                new_range_list.append(self.Range(pivot, left_end, set(states)))
                pivot = left_end
                for cat in left.categories:
                    states.remove(cat)
                continue
            else:
                new_range_list.append(self.Range(pivot, right_begin, set(states)))
                pivot = right_begin
                states.extend(right.categories)
                left_chain.put(right)
                left_chain.put(left)
                right_chain.pop(0)
        self.range_list = []
        _range = new_range_list[0]
        for irange in new_range_list[1:]:
            if irange.low == _range.high and irange.categories == _range.categories:
                _range = self.Range(_range.low, irange.high, _range.categories)
github WorksApplications / SudachiPy / sudachipy / dictionarylib / charactercategory.py View on Github external
pivot = left_end
                for cat in left.categories:
                    states.remove(cat)
                continue
            else:
                new_range_list.append(self.Range(pivot, right_begin, set(states)))
                pivot = right_begin
                states.extend(right.categories)
                left_chain.put(right)
                left_chain.put(left)
                right_chain.pop(0)
        self.range_list = []
        _range = new_range_list[0]
        for irange in new_range_list[1:]:
            if irange.low == _range.high and irange.categories == _range.categories:
                _range = self.Range(_range.low, irange.high, _range.categories)
            else:
                self.range_list.append(_range)
                _range = irange
        self.range_list.append(_range)