python 中langid的阅读。

langid的源码结构

首先我直接进class中 from_modelstring,from_modelpath方法

@classmethod  def from_modelstring(cls, string, *args, **kwargs):    b = base64.b64decode(string)#解码    '''is the string to decode.  Optional altchars must be a string of at least    length 2 (additional characters are ignored) which specifies the    alternative alphabet used instead of the '+' and '/' characters'''    z = bz2.decompress(b)#解压    """            decompress(data) -> decompressed data            Decompress data in one shot. If you want to decompress data sequentially,            use an instance of BZ2Decompressor instead.            """    model = loads(z)    """ loads(string) -- Load a pickle from the given string """    nb_ptc, nb_pc, nb_classes, tk_nextmove, tk_output = model #get value from model      #as I konw nb_pc ,nb_ptc is list        nb_numfeats = int(len(nb_ptc) / len(nb_pc))#retuen int        # reconstruct pc and ptc        nb_pc = np.array(nb_pc)#[1,2,3]        nb_ptc = np.array(nb_ptc).reshape(nb_numfeats, len(nb_pc))#nb_numfeats is col len(nb_pc) is row        '''look like nb_ptc=array([[0, 4, 3],                                    [2, 1, 5]])'''        return cls(nb_ptc, nb_pc, nb_numfeats, nb_classes, tk_nextmove, tk_output, *args, **kwargs)    @classmethod    def from_modelpath(cls, path, *args, **kwargs):        with open(path) as f:            return cls.from_modelstring(f.read().encode(), *args, **kwargs)    def __init__(self, nb_ptc, nb_pc, nb_numfeats, nb_classes, tk_nextmove, tk_output,                 norm_probs=NORM_PROBS):        self.nb_ptc = nb_ptc        self.nb_pc = nb_pc        self.nb_numfeats = nb_numfeats        self.nb_classes = nb_classes        self.tk_nextmove = tk_nextmove        self.tk_output = tk_output        #dirctily to kown value        if norm_probs:            def norm_probs(pd):                """                Renormalize log-probs into a proper distribution (sum 1)                The technique for dealing with underflow is described in                http://jblevins.org/log/log-sum-exp                """                # Ignore overflow when computing the exponential. Large values                # in the exp produce a result of inf, which does not affect                # the correctness of the calculation (as 1/x->0 as x->inf).                # On Linux this does not actually trigger a warning, but on                # Windows this causes a RuntimeWarning, so we explicitly                # suppress it.                with np.errstate(over='ignore'):                    pd = (1 / np.exp(pd[None, :] - pd[:, None]).sum(1))                return pd        else:            def norm_probs(pd):                return pd        self.norm_probs = norm_probs        # Maintain a reference to the full model, in case we change our language set        # multiple times.        self.__full_model = nb_ptc, nb_pc, nb_classes

从作者训练的模型中导入数据，然后看set_languages

    def set_languages(self, langs=None):        logger.debug("restricting languages to: %s", langs)        # Unpack the full original model. This is needed in case the language set        # has been previously trimmed, and the new set is not a subset of the current        # set.        nb_ptc, nb_pc, nb_classes = self.__full_model        if langs is None:            self.nb_classes = nb_classes            self.nb_ptc = nb_ptc            self.nb_pc = nb_pc        else:            # We were passed a restricted set of languages. Trim the arrays accordingly            # to speed up processing.            for lang in langs:                if lang not in nb_classes:                    raise ValueError("Unknown language code %s" % lang)            subset_mask = np.fromiter((l in langs for l in nb_classes), dtype=bool)            self.nb_classes = [c for c in nb_classes if c in langs]            self.nb_ptc = nb_ptc[:, subset_mask]            self.nb_pc = nb_pc[subset_mask]

设置可选语言检测语句
ex

langid.set_languages('zh','en')

再进入instance2fv

    def instance2fv(self, text):        """        Map an instance into the feature space of the trained model.        """        if (sys.version_info > (3, 0)):            # Python3            if isinstance(text, str):                text = text.encode('utf8')        else:            # Python2            if isinstance(text, unicode):                text = text.encode('utf8')            # Convert the text to a sequence of ascii values            text = map(ord, text)#a list of ord like [125,54,74,65]        arr = np.zeros((self.nb_numfeats,), dtype='uint32')#array of uint32        # Count the number of times we enter each state        state = 0        statecount = defaultdict(int) #dic for deafult null        for letter in text:            state = self.tk_nextmove[(state << 8) + letter]            statecount[state] += 1            #return {ja:3,zh:4,en:0}        # Update all the productions corresponding to the state        for state in statecount:            for index in self.tk_output.get(state, []):                arr[index] += statecount[state]        return arr