89af3b3db4dd73cc1187c774beb5685eb18e712a,conceptnet5/language/token_utils.py,,,#,38

Before Change

// A token is a sequence that matches TOKEN_RANGE, or multiple such sequences
// joined only by an apostrophe (as in the word "can"t").
TOKEN_RE = re.compile("{0}+(?:"{0}+)*".format(TOKEN_RANGE))


def simple_tokenize(text):

After Change

// This expression scans through a reversed string to find segments of
// camel-cased text. Comments show what these mean, forwards, in preference
// order:
CAMEL_RE = re.compile(r
    ^( [A-Z]+                 // A string of all caps, such as an acronym
     | [^A-Z0-9 _]+[A-Z _]    // A single capital letter followed by lowercase
                              //   letters, or lowercase letters on their own
                              //   after a word break
     | [^A-Z0-9 _]*[0-9.]+    // A number, possibly followed by lowercase
                              //   letters
     | [ _]+                  // Extra word breaks (spaces or underscores)
     | [^A-Z0-9]*[^A-Z0-9_ ]+ // Miscellaneous symbols, possibly with lowercase
                              //   letters after them
     )
, re.VERBOSE)


def un_camel_case(text):
    r

Instances