Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def ifd_taglist_entity(txt):
i = IFD_Tagset(c="entity", x=txt)
return [(str(i), 1.0)]
def tag_stream(sentence_stream: Iterable[Iterable[Dict[str, Any]]]) -> Iterator[str]:
""" Generator for tag stream from a token stream """
for sent in sentence_stream:
if not sent:
continue
# For each sentence, start and end with empty strings
for _ in range(n - 1):
yield ""
for t in sent:
tag = None
# Skip punctuation
if t.get("k", TOK.WORD) != TOK.PUNCTUATION:
canonicalize_token(t)
tag = str(IFD_Tagset(t))
if tag:
self.lemma_cnt[t["x"]][tag] += 1
if tag:
yield tag
for _ in range(n - 1):
yield ""
def ifd_tag(kind, txt, m):
i = IFD_Tagset(
k=TOK.descr[kind],
c=m.ordfl,
t=m.ordfl,
f=m.fl,
x=txt,
s=m.stofn,
b=m.beyging,
)
return str(i)
def ifd_tag_person(txt, p):
i = IFD_Tagset(
k="PERSON",
c="person",
g=p.gender,
x=txt,
s=p.name,
t="person_" + p.gender + "_" + p.case,
)
return str(i)