
    xnh(                         d dl Z d dlZddlmZ ddlmZmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZ  G d de      Zej'                          y)    N   )messages)unicode_blockUNICODE_BASIC_LATINUNICODE_LATIN_1_SUPPLEMENTUNICODE_LATIN_EXTENDED_BUNICODE_GENERAL_PUNCTUATIONUNICODE_ARABIC!UNICODE_LATIN_EXTENDED_ADDITIONALUNICODE_HIRAGANAUNICODE_KATAKANAUNICODE_BOPOMOFOUNICODE_BOPOMOFO_EXTENDEDUNICODE_CJK_UNIFIED_IDEOGRAPHSUNICODE_HANGUL_SYLLABLESc                   ~   e Zd Z ej                  d      ZdZd Zd Zd Z	e
d        Ze
d        Z ej                  d       ej                  d	       ej                  d
       ej                  d       ej                  d      gZ ej                  d      Z ej                  d      Z ej"                  dez   dz   ez   dz   ej$                        Zg  ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d        ej                  d!       ej                  d"       ej                  d#       ej                  d$       ej                  d%       ej                  d&       ej                  d'       ej                  d(       ej                  d)       ej                  d*       ej                  d+       ej                  d,       ej                  d-       ej                  d.       ej                  d/       ej                  d0       ej                  d1       ej                  d2       ej                  d3       ej                  d4       ej                  d5       ej                  d6       ej                  d7       ej                  d8       ej                  d9       ej                  d:       ej                  d;       ej                  d<       ej                  d=       ej                  d>       ej                  d?       ej                  d@       ej                  dA       ej                  dB       ej                  dC       ej                  dD       ej                  dE       ej                  dF       ej                  dG       ej                  dH       ej                  dI       ej                  dJ       ej                  dK       ej                  dL       ej                  dM       ej                  dN       ej                  dO       ej                  dP       ej                  dQ       ej                  dR       ej                  dS       ej                  dT       ej                  dU       ej                  dV       ej                  dW       ej                  dX       ej                  dY       ej                  dZ       ej                  d[       ej                  d\       ej                  d]       ej                  d^       ej                  d_       ej                  d`       ej                  da       ej                  db       ej                  dc       ej                  dd       ej                  de       ej                  df       ej                  dg       ej                  dh       ej                  di       ej                  dj       ej                  dk       ej                  dl       ej                  dm       ej                  dn       ej                  do       ej                  dp       ej                  dq       ej                  dr       ej                  ds       ej                  dt       ej                  du       ej                  dv       ej                  dw       ej                  dx       ej                  dy       ej                  dz       ej                  d{       ej                  d|       ej                  d}       ej                  d~       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d       ej                  d      Zi Ze
d        Zy)NGramzNGram.LATIN1_EXCLUDE   c                      d| _         d| _        y )N F)gramscapitalword)selfs    W/var/www/html/profi_bot/bot/venv/lib/python3.12/site-packages/langdetect/utils/ngram.py__init__zNGram.__init__   s    
     c                 l   | j                  |      }| j                  d   }|dk(  rd| _        d| _        |dk(  r7yt        | j                        | j                  k\  r| j                  dd | _        | xj                  |z  c_        |j                         r|j                         rd| _        yyd| _        y)z%Append a character into ngram buffer.r   FNr   T)	normalizer   r   lenN_GRAMisupper)r   ch	last_chars      r   add_charzNGram.add_char   s    ^^BJJrN	DJ$DSy_+ABDJ

b
::<  "#'  #  %Dr   c                     | j                   ry|dk  s'|| j                  kD  st        | j                        |k  ry|dk(  r| j                  d   }|dk(  ry|S | j                  | d S )zGet n-gram.Nr   r   r   )r   r!   r    r   )r   nr#   s      r   getz	NGram.get2   sf    q5AOs4::':6BBSyI::qbc?"r   c                    t        |      }|t        k(  r|dk  sd|cxk  rdk  sn d|k  rd}|S |t        k(  r"| j                  j	                  |      dk\  rd}|S |t
        k(  r\|t        j                  d      k(  rt        j                  d      }|t        j                  d	      k(  rt        j                  d
      }|S |t        k(  rd}|S |t        k(  r/|t        j                  d      k(  rt        j                  d      }|S |t        k(  r/|t        j                  d      k\  rt        j                  d      }|S |t        k(  rt        j                  d      }|S |t        k(  rt        j                  d      }|S |t        t        fv rt        j                  d      }|S |t        k(  r| j                   j#                  ||      }|S |t$        k(  rt        j                  d      }|S )NAZazr   r   u   șu   şu   țu   ţu   یu   يu   Ạu   ểu   あu   アu   ㄅu   가)r   r   r   LATIN1_EXCLUDEDfindr   sixur	   r
   r   r   r   r   r   r   CJK_MAPr(   r   )clsr#   blocks      r   r   zNGram.normalize@   s   b!''CxC"NsNsRx8 	7 00""''+q02 	1 ..SUU8_$UU8_SUU8_$UU8_& 	% 11B" 	! n$SUU8_$UU8_ 	 77SUU8_$UU8_ 	 &&xB 	 &&xB 	 ')BCCxB
 		 44R(B 	 ..xB	r   c                 F      fd} j                   j                  ||      S )zfNormalizer for Vietnamese.
        Normalize Alphabet + Diacritical Mark(U+03xx) into U+1Exx.
        c                     j                   j                  | j                  d            }j                  j                  | j                  d            }j                  |   |   S )Nr      )TO_NORMALIZE_VI_CHARSr/   groupDMARK_CLASSNORMALIZED_VI_CHARS)malphabetdmarkr3   s      r   replz NGram.normalize_vi.<locals>.replh   sS    0055aggajAHOO((4E**51(;;r   )ALPHABET_WITH_DMARKsub)r3   textr?   s   `  r   normalize_vizNGram.normalize_vic   s#    
	< &&**466r   NORMALIZED_VI_CHARS_0300NORMALIZED_VI_CHARS_0301NORMALIZED_VI_CHARS_0303NORMALIZED_VI_CHARS_0309NORMALIZED_VI_CHARS_0323r8   r:   z([z])([z])zNGram.KANJI_1_0zNGram.KANJI_1_2zNGram.KANJI_1_4zNGram.KANJI_1_8zNGram.KANJI_1_11zNGram.KANJI_1_12zNGram.KANJI_1_13zNGram.KANJI_1_14zNGram.KANJI_1_16zNGram.KANJI_1_18zNGram.KANJI_1_22zNGram.KANJI_1_27zNGram.KANJI_1_29zNGram.KANJI_1_31zNGram.KANJI_1_35zNGram.KANJI_2_0zNGram.KANJI_2_1zNGram.KANJI_2_4zNGram.KANJI_2_9zNGram.KANJI_2_10zNGram.KANJI_2_11zNGram.KANJI_2_12zNGram.KANJI_2_13zNGram.KANJI_2_15zNGram.KANJI_2_16zNGram.KANJI_2_18zNGram.KANJI_2_21zNGram.KANJI_2_22zNGram.KANJI_2_23zNGram.KANJI_2_28zNGram.KANJI_2_29zNGram.KANJI_2_30zNGram.KANJI_2_31zNGram.KANJI_2_32zNGram.KANJI_2_35zNGram.KANJI_2_36zNGram.KANJI_2_37zNGram.KANJI_2_38zNGram.KANJI_3_1zNGram.KANJI_3_2zNGram.KANJI_3_3zNGram.KANJI_3_4zNGram.KANJI_3_5zNGram.KANJI_3_8zNGram.KANJI_3_9zNGram.KANJI_3_11zNGram.KANJI_3_12zNGram.KANJI_3_13zNGram.KANJI_3_15zNGram.KANJI_3_16zNGram.KANJI_3_18zNGram.KANJI_3_19zNGram.KANJI_3_22zNGram.KANJI_3_23zNGram.KANJI_3_27zNGram.KANJI_3_29zNGram.KANJI_3_30zNGram.KANJI_3_31zNGram.KANJI_3_32zNGram.KANJI_3_35zNGram.KANJI_3_36zNGram.KANJI_3_37zNGram.KANJI_3_38zNGram.KANJI_4_0zNGram.KANJI_4_9zNGram.KANJI_4_10zNGram.KANJI_4_16zNGram.KANJI_4_17zNGram.KANJI_4_18zNGram.KANJI_4_22zNGram.KANJI_4_24zNGram.KANJI_4_28zNGram.KANJI_4_34zNGram.KANJI_4_39zNGram.KANJI_5_10zNGram.KANJI_5_11zNGram.KANJI_5_12zNGram.KANJI_5_13zNGram.KANJI_5_14zNGram.KANJI_5_18zNGram.KANJI_5_26zNGram.KANJI_5_29zNGram.KANJI_5_34zNGram.KANJI_5_39zNGram.KANJI_6_0zNGram.KANJI_6_3zNGram.KANJI_6_9zNGram.KANJI_6_10zNGram.KANJI_6_11zNGram.KANJI_6_12zNGram.KANJI_6_16zNGram.KANJI_6_18zNGram.KANJI_6_20zNGram.KANJI_6_21zNGram.KANJI_6_22zNGram.KANJI_6_23zNGram.KANJI_6_25zNGram.KANJI_6_28zNGram.KANJI_6_29zNGram.KANJI_6_30zNGram.KANJI_6_32zNGram.KANJI_6_34zNGram.KANJI_6_35zNGram.KANJI_6_37zNGram.KANJI_6_39zNGram.KANJI_7_0zNGram.KANJI_7_3zNGram.KANJI_7_6zNGram.KANJI_7_7zNGram.KANJI_7_9zNGram.KANJI_7_11zNGram.KANJI_7_12zNGram.KANJI_7_13zNGram.KANJI_7_16zNGram.KANJI_7_18zNGram.KANJI_7_19zNGram.KANJI_7_20zNGram.KANJI_7_21zNGram.KANJI_7_23zNGram.KANJI_7_25zNGram.KANJI_7_28zNGram.KANJI_7_29zNGram.KANJI_7_32zNGram.KANJI_7_33zNGram.KANJI_7_35zNGram.KANJI_7_37c                 \    | j                   D ]  }|d   }|D ]  }|| j                  |<     y )Nr   )	CJK_CLASSr2   )r3   cjk_listrepresentativer#   s       r   _init_cjk_mapzNGram._init_cjk_map   s0    H%a[N"0B  &r   N)__name__
__module____qualname__r   
get_stringr.   r!   r   r%   r(   classmethodr   rC   r;   r8   r:   recompileUNICODEr@   rJ   r2   rM    r   r   r   r      s3   )h))*@AOF!%&#    D 7 7 	67676767679 0H//0GH%(%%m4K$"**$$v-;dB



~1-.~1-.~1 	-.~1 	-.	~1
 	./~1 	./~1 	./~1 	./~1 	./~1 	./~1 	./~1 	./~1 	./~1 	./~1 	./~1  	-.!~1" 	-.#~1$ 	-.%~1& 	-.'~1( 	./)~1* 	./+~1, 	./-~1. 	.//~10 	./1~12 	./3~14 	./5~16 	./7~18 	./9~1: 	./;~1< 	./=~1> 	./?~1@ 	./A~1B 	./C~1D 	./E~1F 	./G~1H 	./I~1J 	./K~1L 	./M~1N 	-.O~1P 	-.Q~1R 	-.S~1T 	-.U~1V 	-.W~1X 	-.Y~1Z 	-.[~1\ 	./]~1^ 	./_~1` 	./a~1b 	./c~1d 	./e~1f 	./g~1h 	./i~1j 	./k~1l 	./m~1n 	./o~1p 	./q~1r 	./s~1t 	./u~1v 	./w~1x 	./y~1z 	./{~1| 	./}~1~ 	./~1@ 	-.A~1B 	-.C~1D 	./E~1F 	./G~1H 	./I~1J 	./K~1L 	./M~1N 	./O~1P 	./Q~1R 	./S~1T 	./U~1V 	./W~1X 	./Y~1Z 	./[~1\ 	./]~1^ 	./_~1` 	./a~1b 	./c~1d 	./e~1f 	./g~1h 	./i~1j 	-.k~1l 	-.m~1n 	-.o~1p 	./q~1r 	./s~1t 	./u~1v 	./w~1x 	./y~1z 	./{~1| 	./}~1~ 	./~1@ 	./A~1B 	./C~1D 	./E~1F 	./G~1H 	./I~1J 	./K~1L 	./M~1N 	./O~1P 	./Q~1R 	./S~1T 	-.U~1V 	-.W~1X 	-.Y~1Z 	-.[~1\ 	-.]~1^ 	./_~1` 	./a~1b 	./c~1d 	./e~1f 	./g~1h 	./i~1j 	./k~1l 	./m~1n 	./o~1p 	./q~1r 	./s~1t 	./u~1v 	./w~1x 	./y~1z 	./{~1| 	./}~1I@ G1 1r   r   )rS   r0    r   r   r   r   r   r	   r
   r   r   r   r   r   r   r   objectr   rM   rV   r   r   <module>rY      s?    	 
    "k1F k1Z    r   