python正向最大匹配分词和逆向最大匹配分词的实例

# -*- coding:utf-8 -*-
 
CODEC=\'utf-8\'
 
def u(s, encoding):
  \'converted other encoding to unicode encoding\'
  if isinstance(s, unicode):
    return s
  else:
    return unicode(s, encoding)
 
def fwd_mm_seg(wordDict, maxLen, str):
  \'forward max match segment\'
  wordList = []
  segStr = str
  segStrLen = len(segStr)
  for word in wordDict:
    print \'word: \', word
  print \"\\n\"
  while segStrLen > 0:
    if segStrLen > maxLen:
      wordLen = maxLen
    else:
      wordLen = segStrLen
    subStr = segStr[0:wordLen]
    print \"subStr: \", subStr
    while wordLen > 1:
      if subStr in wordDict:
        print \"subStr1: %r\" % subStr
        break
      else:
        print \"subStr2: %r\" % subStr
        wordLen = wordLen - 1
        subStr = subStr[0:wordLen]
#      print \"subStr3: \", subStr
    wordList.append(subStr)
    segStr = segStr[wordLen:]
    segStrLen = segStrLen - wordLen
  for wordstr in wordList:
    print \"wordstr: \", wordstr
  return wordList
    
      
def main():
  fp_dict = open(\'words.dic\')
  wordDict = {}
  for eachWord in fp_dict:
    wordDict[u(eachWord.strip(), \'utf-8\')] = 1
  segStr = u\'你好世界hello world\'
  print segStr
  wordList = fwd_mm_seg(wordDict, 10, segStr)
  print \"==\".join(wordList)
  
 
if __name__ == \'__main__\':
  main()
  

# -*- coding:utf-8 -*-
 
 
def u(s, encoding):
  \'converted other encoding to unicode encoding\'
  if isinstance(s, unicode):
    return s
  else:
    return unicode(s, encoding)
 
CODEC=\'utf-8\'
 
def bwd_mm_seg(wordDict, maxLen, str):
  \'forward max match segment\'
  wordList = []
  segStr = str
  segStrLen = len(segStr)
  for word in wordDict:
    print \'word: \', word
  print \"\\n\"
  while segStrLen > 0:
    if segStrLen > maxLen:
      wordLen = maxLen
    else:
      wordLen = segStrLen
    subStr = segStr[-wordLen:None]
    print \"subStr: \", subStr
    while wordLen > 1:
      if subStr in wordDict:
        print \"subStr1: %r\" % subStr
        break
      else:
        print \"subStr2: %r\" % subStr
        wordLen = wordLen - 1
        subStr = subStr[-wordLen:None]
#      print \"subStr3: \", subStr
    wordList.append(subStr)
    segStr = segStr[0: -wordLen]
    segStrLen = segStrLen - wordLen
  wordList.reverse()
  for wordstr in wordList:
    print \"wordstr: \", wordstr
  return wordList
    
      
def main():
  fp_dict = open(\'words.dic\')
  wordDict = {}
  for eachWord in fp_dict:
    wordDict[u(eachWord.strip(), \'utf-8\')] = 1
  segStr = ur\'你好世界hello world\'
  print segStr
  wordList = bwd_mm_seg(wordDict, 10, segStr)
  print \"==\".join(wordList)
 
if __name__ == \'__main__\':
  main()