5d4a480c48a242c1d8051be502ecb98a9b273e9c,tests/candidates/test_candidates.py,,test_ngram_split,#Any#,33

Before Change

    pdf_path = "tests/data/pdf/112823.pdf"

    // Parsing
    logger.info("Parsing...")
    doc_preprocessor = HTMLDocPreprocessor(docs_path, max_docs=max_docs)
    corpus_parser = Parser(
        structural=True, lingual=True, visual=True, pdf_path=pdf_path
    )

After Change

    // When more than one split_token appears.
    sent.text = "New/Text-Word"
    sent.words = ["New/Text-Word"]
    result = list(ngrams.apply(sent))

    assert len(result) == 3
    assert result[0].get_span() == "New/Text-Word"
    assert result[1].get_span() == "New"

Instances