How to use the sudachipy.plugin.oov.MeCabOovPlugin.CategoryInfo function in SudachiPy

To help you get started, we’ve selected a few SudachiPy examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github WorksApplications / SudachiPy / tests / plugin / test_mecab_oov_plugin.py View on Github external
def test_provide_oov110(self):
        cinfo = MeCabOovPlugin.CategoryInfo()
        cinfo.type_ = CategoryType.KANJI
        cinfo.is_invoke = True
        cinfo.is_group = True
        cinfo.length = 0
        self.plugin.categories[CategoryType.KANJI] = cinfo

        mock_inputtext.set_category_type(0, 3, CategoryType.KANJI)

        nodes = self.plugin.provide_oov(self.mocked_input_text, 0, False)
        self.assertEqual(1, len(nodes))

        n = nodes[0]
        self.assertEqual('あいう', n.get_word_info().surface)
        self.assertEqual(3, n.get_word_info().length())
        self.assertEqual(1, n.get_word_info().pos_id)
github WorksApplications / SudachiPy / tests / plugin / test_mecab_oov_plugin.py View on Github external
def test_provide_oov_without_oov_list(self):
        cinfo = MeCabOovPlugin.CategoryInfo()
        cinfo.type_ = CategoryType.HIRAGANA
        cinfo.is_invoke = False
        cinfo.is_group = True
        cinfo.length = 0
        self.plugin.categories[CategoryType.HIRAGANA] = cinfo

        mock_inputtext.set_category_type(0, 3, CategoryType.HIRAGANA)

        nodes = self.plugin.provide_oov(self.mocked_input_text, 0, False)
        self.assertEqual(0, len(nodes))
github WorksApplications / SudachiPy / tests / plugin / test_mecab_oov_plugin.py View on Github external
def test_read_oov_with_too_few_columns(self):
        input_ = os.path.join(self.test_dir, 'test.txt')
        with open(input_, 'w') as wf:
            wf.write("DEFAULT,1,2,3\n")
        plugin = MeCabOovPlugin()
        plugin.categories[CategoryType.DEFAULT] = MeCabOovPlugin.CategoryInfo()
        with self.assertRaises(ValueError) as cm:
            plugin.read_oov(input_, mock_grammar.mocked_grammar)
        self.assertEqual('invalid format at line 1', cm.exception.args[0])
github WorksApplications / SudachiPy / tests / plugin / test_mecab_oov_plugin.py View on Github external
def test_read_oov(self):
        oov = os.path.join(self.test_dir, 'test.txt')
        with open(oov, 'w') as wf:
            wf.write("DEFAULT,1,2,3,補助記号,一般,*,*,*,*\n")
            wf.write("DEFAULT,3,4,5,補助記号,一般,*,*,*,*\n")
        plugin = MeCabOovPlugin()
        plugin.categories[CategoryType.DEFAULT] = MeCabOovPlugin.CategoryInfo()
        plugin.read_oov(oov, mock_grammar.mocked_grammar)
        self.assertEqual(1, len(plugin.oov_list))
        self.assertEqual(2, len(plugin.oov_list[CategoryType.DEFAULT]))
        self.assertEqual(1, plugin.oov_list[CategoryType.DEFAULT][0].left_id)
        self.assertEqual(2, plugin.oov_list[CategoryType.DEFAULT][0].right_id)
        self.assertEqual(3, plugin.oov_list[CategoryType.DEFAULT][0].cost)
        self.assertEqual(0, plugin.oov_list[CategoryType.DEFAULT][0].pos_id)
github WorksApplications / SudachiPy / tests / plugin / test_mecab_oov_plugin.py View on Github external
def test_provide_oov006(self):
        cinfo = MeCabOovPlugin.CategoryInfo()
        cinfo.type_ = CategoryType.KANJI
        cinfo.is_invoke = False
        cinfo.is_group = False
        cinfo.length = 6
        self.plugin.categories[CategoryType.KANJI] = cinfo

        mock_inputtext.set_category_type(0, 3, CategoryType.KANJI)

        nodes = self.plugin.provide_oov(self.mocked_input_text, 0, False)
        self.assertEqual(3, len(nodes))

        n = nodes[0]
        self.assertEqual('あ', n.get_word_info().surface)
        self.assertEqual(1, n.get_word_info().length())
        self.assertEqual(1, n.get_word_info().pos_id)
github WorksApplications / SudachiPy / tests / plugin / test_mecab_oov_plugin.py View on Github external
def test_read_oov_with_category_not_in_character_property(self):
        input_ = os.path.join(self.test_dir, 'test.txt')
        with open(input_, 'w') as wf:
            wf.write("ALPHA,1,2,3,補助記号,一般,*,*,*,*\n")
        plugin = MeCabOovPlugin()
        plugin.categories[CategoryType.DEFAULT] = MeCabOovPlugin.CategoryInfo()
        with self.assertRaises(ValueError) as cm:
            plugin.read_oov(input_, mock_grammar.mocked_grammar)
        self.assertEqual('`ALPHA` is undefined at line 1', cm.exception.args[0])
github WorksApplications / SudachiPy / tests / plugin / test_mecab_oov_plugin.py View on Github external
def test_provide_oov_multi_oov(self):
        cinfo = MeCabOovPlugin.CategoryInfo()
        cinfo.type_ = CategoryType.KANJINUMERIC
        cinfo.is_invoke = False
        cinfo.is_group = True
        cinfo.length = 0
        self.plugin.categories[CategoryType.KANJINUMERIC] = cinfo

        mock_inputtext.set_category_type(0, 3, CategoryType.KANJINUMERIC)

        nodes = self.plugin.provide_oov(self.mocked_input_text, 0, False)
        self.assertEqual(2, len(nodes))

        n = nodes[0]
        self.assertEqual('あいう', n.get_word_info().surface)
        self.assertEqual(3, n.get_word_info().length())
        self.assertEqual(1, n.get_word_info().pos_id)
github WorksApplications / SudachiPy / tests / plugin / test_mecab_oov_plugin.py View on Github external
def test_provide_oov112(self):
        cinfo = MeCabOovPlugin.CategoryInfo()
        cinfo.type_ = CategoryType.KANJI
        cinfo.is_invoke = True
        cinfo.is_group = True
        cinfo.length = 2
        self.plugin.categories[CategoryType.KANJI] = cinfo

        mock_inputtext.set_category_type(0, 3, CategoryType.KANJI)

        nodes = self.plugin.provide_oov(self.mocked_input_text, 0, False)
        self.assertEqual(3, len(nodes))

        n = nodes[0]
        self.assertEqual('あいう', n.get_word_info().surface)
        self.assertEqual(3, n.get_word_info().length())
        self.assertEqual(1, n.get_word_info().pos_id)
github WorksApplications / SudachiPy / tests / plugin / test_mecab_oov_plugin.py View on Github external
def test_provide_oov100(self):
        cinfo = MeCabOovPlugin.CategoryInfo()
        cinfo.type_ = CategoryType.KANJI
        cinfo.is_invoke = True
        cinfo.is_group = False
        cinfo.length = 0
        self.plugin.categories[CategoryType.KANJI] = cinfo

        mock_inputtext.set_category_type(0, 3, CategoryType.KANJI)

        nodes = self.plugin.provide_oov(self.mocked_input_text, 0, False)
        self.assertEqual(0, len(nodes))

        nodes = self.plugin.provide_oov(self.mocked_input_text, 0, True)
        self.assertEqual(0, len(nodes))