
    /j                        d Z ddlZddlZddlZddlZddlZddlmZ ddlZddl	m
Z
 ddlmZ  ej                  ej                  d        ej                  d      Zd	ed
efdZdefdZd Zded
e
fdZdefdZd Zedk(  r e        yy)u  
Phase 1 — Export: 从 v0.17.2 按知识库导出文件.

1. 连接 v0.17.2 MySQL，读取 Knowledgebase → Document → File 关联
2. 从 MinIO 下载文件到本地
3. 输出 manifest.json（kb_id 为主键，避免同名冲突）

Usage:
    python export_files.py -c config.yaml
    N)Path)Minio)YAMLz'%(asctime)s [%(levelname)s] %(message)s)levelformatexportpathreturnc                     t        dd      }t        |       5 }|j                  |      cd d d        S # 1 sw Y   y xY w)NsafeT)typpure)r   openload)r	   yamlfs      export_files.pyload_configr      s7    F&D	d qyy|  s   4=cfgc           	          | d   }t        j                  |d   |d   |d   |d   |d   dt         j                  j                        }t        j                  d	|d   |d   |d          |S )
Nmysqlhostportuserpassworddatabaseutf8mb4)r   r   r   r   r   charsetcursorclasszMySQL: %s:%d/%s)pymysqlconnectcursors
DictCursorloginfo)r   mcconns      r   connect_mysqlr(   &   sn    	WB??Zbjr&zJ"Z.w'A'AD
 HH6
BvJ:GK    c                     | j                         5 }|j                  ||       |j                         cd d d        S # 1 sw Y   y xY w)N)cursorexecutefetchall)r'   sqlparamscurs       r   fetchr1   1   s9    	 #C ||~  s	   "=Ac           	          | d   }t        |d   |d   |d   |j                  dd            }t        j                  d|d          |S )	Nstorager   
access_key
secret_keysecureF)r4   r5   r6   z	MinIO: %s)r   getr$   r%   )r   scclients      r   connect_minior:   7   sO    	YB2f:"\*: .rvvh7NPFHH["V*%Mr)   c           
         t        | d         }|dz  }|j                  dd       t        | d         }t        | d         }t	        |d      }t
        j                  dt        |             g }d}|D ]  }|d	   }	|d
   }
t	        |d|	      }|s'|j                  |	|
|j                  dd      g d       D|D cg c]  }|d	   	 }}|D ci c]  }|d	   |
 }}dj                  dgt        |      z        }t	        |d| dg| }t        |D ch c]  }|d   	 c}      }i }|rBdj                  dgt        |      z        }t	        |d| dg| }|D ci c]  }|d	   |
 }}i }|D ]$  }|j                  |d         }|s|d
   ||d   <   & |
 d|	d d  }||z  }|j                  dd       g }|D ]  }|d	   }|j                  |      xs |j                  d
      xs | d}||z  }d}t        j                  j                  |      \  } }!|j                         r"||  d| |! z  }|dz  }|j                         r"|	}"|j                  d      }#|#st
        j!                  d|       	 |j#                  |"|#t%        |             |j                  |j&                  t%        |j)                  |            d       |dz  } |j                  |	|
|j                  dd      |d       t
        j                  d|
t        |              |j/                          |d z  }%t1        |%d!d"#      5 }t3        j4                  ||d$d%&       d d d        t
        j                  d'|%       t
        j                  d(t        |      |       y c c}w c c}w c c}w c c}w # t*        $ r#}$t
        j-                  d|"|#|$       Y d }$~$d }$~$ww xY w# 1 sw Y   xY w))Ntemp_dirfilesT)parentsexist_oksourcezX
        SELECT id, name, description
        FROM knowledgebase WHERE status = '1'
    zKB count: %dr   idnamezq
            SELECT id, name, location, type
            FROM document WHERE kb_id = %s AND status = '1'
        description )kb_idkb_namekb_descriptionr=   ,z%sz^
            SELECT document_id, file_id
            FROM file2document WHERE document_id IN (z
)
        file_idz8
                SELECT id, name FROM file WHERE id IN (z)
            document_id_   z.bin   locationzdoc %s has no location, skip)rB   
local_pathzdownload failed %s/%s: %szKB %s: %d fileszmanifest.jsonwzutf-8)encodingF   )ensure_asciiindentu   manifest → %szDone: %d KBs, %d files)r   mkdirr(   r:   r1   r$   r%   lenappendr7   joinlistosr	   splitextexistswarningfget_objectstrrB   relative_to	Exceptionerrorcloser   jsondump)&r   r<   file_dirr'   minio_clientkbsmanifesttotal_fileskbrE   rF   docsddoc_idsdoc_mapplaceholdersf2drfile_idsfile_mapfpr=   r   doc_file_namessafe_dirkb_file_dirfiles_manifestdocdoc_iddoc_namerO   counterstemsuffixbucketkeyemanifest_paths&                                         r   runr   A   sn   C
O$H'!HNN4$N/X'D X/L   	C HH^SX&HK SB4V* T  
 OOe-/VVM2-FQSU V$()q1T7))'+,!1T7A:,, xxW 56D 66B^ D	  
 s3!9344&3x=01B$ %88:t <! E -22q$
2H2 *, 	=AQy\*A34V9q/0	= Yabq	{+)$6 	GCYF%))&1WSWWV_W6(RVH$x/J G77++H5LD&##%(dV1WIfX+FF
1 ##%
 F''*%C:FCG((c*oF%%&OO"%j&<&<X&F"G'  q 5	G< 	 ff]B7#	
 	 	"GS-@AgSBj 	JJL .M	mS7	3 =q		(AE!<=HH.HH%s8}kB[ *, 4 3T  G		5vsAFFG= =s=   <NNN$
N);AN.O.	O7OOO&c                      t        j                         } | j                  ddd       | j                         }t	        t        |j                               y )Nz-cz--configT)required)argparseArgumentParseradd_argument
parse_argsr   r   config)pargss     r   mainr      s>    !ANN4dN3<<>DDKK !r)   __main__)__doc__r   rd   loggingrZ   syspathlibr   r    minior   ruamel.yamlr   basicConfigINFO	getLoggerr$   r_   dictr   r(   r1   r:   r   r   __name__ r)   r   <module>r      s   	    	 
       ',,/X Yg!
c d t t  nCT nCb" zF r)   