How to use the smt.db.createdb.Sentence function in smt

To help you get started, we’ve selected a few smt examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github kenkov / smt / jec_basic_sentence / jec_createdb.py View on Github external
# create session
    Session = sessionmaker(bind=engine)
    session = Session()

    # get sentence from excel file
    wb = xlrd.open_workbook(excel_file)
    sheets = wb.sheets()
    s = sheets[0]

    with ProgressLine(title="inserting items..."):
        for j in xrange(s.nrows):
            item = []
            for i in xrange(1, s.ncols - 1):
                item.append(s.cell(j, i).value)
            sentence = Sentence(lang1=item[0], lang2=item[1])
            # add items
            session.add(sentence)
        session.commit()
github kenkov / smt / smt / db / createngramdb.py View on Github external
def _create_ngram_count_db(lang, langmethod=lambda x: x,
                           n=3, db="sqilte:///:memory:"):
    engine = create_engine(db)
    # create session
    Session = sessionmaker(bind=engine)
    session = Session()

    query = session.query(Sentence)

    ngram_dic = collections.defaultdict(float)
    for item in query:
        if lang == 1:
            sentences = langmethod(item.lang1).split()
        elif lang == 2:
            sentences = langmethod(item.lang2).split()
        sentences = ["", "<s>"] + sentences + ["</s>"]
        ngrams = ngram(sentences, n)
        for tpl in ngrams:
            ngram_dic[tpl] += 1

    return ngram_dic
github kenkov / smt / jec_basic_sentence / jec_createdb.py View on Github external
def excel_convert(db="sqlite:///:memory:",
                  excel_file="./JEC_basic_sentence_v1-2.xls"):
    engine = create_engine(db)
    # first, remove table
    Sentence.__table__.drop(engine, checkfirst=True)
    # create table
    Sentence.__table__.create(engine)
    print("created table: sentence")

    # create session
    Session = sessionmaker(bind=engine)
    session = Session()

    # get sentence from excel file
    wb = xlrd.open_workbook(excel_file)
    sheets = wb.sheets()
    s = sheets[0]

    with ProgressLine(title="inserting items..."):
        for j in xrange(s.nrows):
            item = []
github kenkov / smt / jec_basic_sentence / jec_createdb.py View on Github external
def excel_convert(db="sqlite:///:memory:",
                  excel_file="./JEC_basic_sentence_v1-2.xls"):
    engine = create_engine(db)
    # first, remove table
    Sentence.__table__.drop(engine, checkfirst=True)
    # create table
    Sentence.__table__.create(engine)
    print("created table: sentence")

    # create session
    Session = sessionmaker(bind=engine)
    session = Session()

    # get sentence from excel file
    wb = xlrd.open_workbook(excel_file)
    sheets = wb.sheets()
    s = sheets[0]

    with ProgressLine(title="inserting items..."):
        for j in xrange(s.nrows):
            item = []
            for i in xrange(1, s.ncols - 1):
                item.append(s.cell(j, i).value)