3
=hLB                 @   sP  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dl
mZ d dl
mZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlZd dlmZ d dlmZ d dlmZ G dd deZdd ZG dd deZ dd Z!dd Z"G dd deZ#d(ddZ$d)d d!Z%d"d# Z&d$d% Z'd&d' Z(dS )*    N)SDI)PAGE)PAGE_READER)GET_FSP_STATUS_FROM_FLAGS)FSP)PARSE_ENCRYPTION_INFO)READ_KEYRING)MYSQLFRM)INODE)INDEX)TABLE)Process)Value)Lockc               @   s   e Zd Zdd Zdd ZdS )PAGE_READER_FRAGMENTc             C   s
   || _ d S )N)filename_pre)selfr    r   2/root/work/ibd2sql-ibd2sql-v2.x/ibd2sql/ibd2sql.py__init__   s    zPAGE_READER_FRAGMENT.__init__c          	   C   sR   t jj| jt|jdd }d}t jj|rNt|d}|jd}W d Q R X |S )N   z.page    rbi @  )	ospathjoinr   strzfillexistsopenread)r   nfilenamedatafr   r   r   r       s    zPAGE_READER_FRAGMENT.readN)__name__
__module____qualname__r   r    r   r   r   r   r      s   r   c             C   sj   xd| j |}|dd dkr P qd}|tjd||d | d 7 }tjd|||d	  \}}}qW |S )
N@   B   s     c   z>h   r   z>LQLr   )r    structunpack)pgpageidr#   offsetZdtypeZtable_idr   r   r   GET_LEAF_PAGE_NO_FROM_SDI#   s    
 "r1   c               @   s   e Zd Zdd Zdd ZdS )IBDBASEc             C   s
  d| _ t|d}|jd}t|dkr6|j|d d S |dd dkr\|j| d d| _ tjd	|d
d \}}}}	}
}t|
| _	| j	d | _
| j	d | _| j| _| j	d | _| j
| j | _| j	d dkrdnd| _| j	d dkrdnd| _| j	d dkrdnd| _| j	d | _|j|d| j
 |j|d| j |j|d| j |j|d| j |j|d| j |j|d| j |j|d| j tjj|| j dkr|j|d |jdd t|j| j| j
| j| _|j  d | _d | _| jrlyJt| jj|}|d | _|d | _|jd| jj  |jd| jj  W n4 t k
rj } z|j|d|d| d S d }~X nX t!| j|| j| j| jd| _"d | _#| jr tt$| j"| jj%| j"| jdkrdnd}y|j& | _#W n0 t k
r } z|j|d| d S d }~X nX d| _ d S )NFr   i   zis too small ...      s    z version may be is too lowz>6L&   >   logical_sizephysical_size
compressed
ENCRYPTION   Tr   SHAREDPOST_ANTELOPEr   zmaybe have been damagedkeyivzmaster_key not inz
exception:)	page_sizer"   
encryptionr>   r?   
COMPRESSED1zget sdi faild)'statusr   r    lenerrorinfor,   r-   r   	fsp_flagsr7   r8   r@   r9   Zcompression_ratior:   r   r<   r=   r   r   getsizewarningseekr   fspcloser>   r?   r   Zencryption_infohex	Exceptionr   r.   sdir1   ZSDI_PAGE_NOget_sdi)r   r"   logkdr$   r#   ZFSP_SPACE_IDZFSP_NOT_USEDZFSP_SIZEZFSP_FREE_LIMITZFSP_SPACE_FLAGSZFSP_FRAG_N_USEDterP   r   r   r   r   /   sl    

 


*zIBDBASE.__init__c             C   s   d S )Nr   )r   r   r   r   testp   s    zIBDBASE.testN)r%   r&   r'   r   rV   r   r   r   r   r2   .   s   Ar2   c             C   sx   | j dd d d }x\tj|D ]N}|| kr0q"tjj|r"t|||}|jr"|jd k	r"t|jdkr"|jd S q"W d S )N#r   z*.ibdr+   )	splitglobr   r   isfiler2   rD   rP   rE   )Z
filename_trR   rS   Zfilename_rer"   ibdbaser   r   r   GET_PARTITION_TABLE_SDIDATAt   s    r\   c             C   sv  i }|dk	r.t |d}t|j }W dQ R X d}|dk	r|jdrt|j|d tjt|j  }}|j|d n|jdr|j|d t |d}tj	|}W dQ R X t
|d	kr|d
 }|j|d n,|jdr|j|d t|||}|jd }g }	xr| D ]h}
|
jdp|
jds2|j|
d qt|
||}|dksR|j rpt
| d
krp|jd|
 qd}d}|j s|j rz|j|
d |
jd}d}|dkr|
d| d }|
|d) }|j|
d n|
dd* d }tjj|r:|j|
d| tjt|j }|j|
d|d d |d d  n<|jd|d |dk	rf|j|
d| |}n|j|
d qn|jjd kr&|jjd
kr&|j|
d!|jj |jr6|j|
d" xr|jD ]h}d|ksd|d krq|j|
d|d d |d d  |	j|
||j|j|j|j||jd# qW qnt
|j}|d
kr|j|
d t|
||}|dkr|dkr|j|
d$ qn|}|j|
d%| n |j|
d|d d |d d  nf|d&krd|jd d kr|jd }n
|jd
 }|j|
d|d d |d d  n|jd'| qn |jd(|
|j|jj|jj q|	j|
||j|j|j|j||jd# qW |	S )+a   
	INPUT:
		filename_list: ibd/frm/sdi file list
		sdi_file: sdi file(ibd/frm/sdi)
		keyring_file: keyring filename
		log: log

	RETURN:
		[
			{
				'filename':xx,
				'sdi':sdi info (dict),
				'key':key,
				'iv':'iv',
				'pagesize': page size(phy),
				'partition_name':''
			},
		]
	Nr   z.frmz	maybe frmzglobal_sdi_info is frm filez.sdiz	maybe sdir   r;   zglobal_sdi_info is sdi filez.ibdz	maybe ibdr   z.pageznot endswith .ibd, skip itz
skip file:zis mysql 5, will get sdi....rW       zis partition tablezwill use frm file:z	ADD TABLE	dd_objectZ
schema_refnamezfrm filez
not existszuse global_sdi_infozhave not sdi info, skip iti8 zmysql version:z	is shared)r"   rP   rA   r>   r?   pagesizepartition_namerH   zcan not find sdi info, skip itzuse global sdir+   zunknown error when read sdiz	skip filere   ) r   r   r    endswithrG   jsonloadsr	   Z_get_sdi_jsonloadrE   r2   rP   rF   rD   r   findr   r   r   rJ   rL   ZFIL_PAGE_PREVZFIL_PAGE_NEXTr<   appendr:   r>   r?   r8   rH   r\   )filename_listZsdi_fileZkeyring_filerR   rS   r$   Zglobal_sdi_infoZsdidatar[   	file_listr"   rd   Zpartition_offsetZfrm_filenamexZ	sdi_countr   r   r   FORMAT_IBD_FILE   s    



"

"
 



"

"ro   c               @   s>   e Zd ZdddZdd Zdd Zd	d
 Zdd Zdd ZdS )IBD2SQLFNc             C   st   || _ || _|| _|| _|| _|r4|d kr4| j| _n<|rJ|d k	rJ| j| _n&| rZ|d krZn| rp|d k	rp| j| _d S )N)r.   r/   forcev_read_page_add1
_read_page_read_page_share_add1_read_page_share)r   r.   r/   rq   rr   r   r   r   r   
  s    

zIBD2SQL.__init__c             C   s,   | j  }t|| jjkr d|fS d|fS d S )NFT)rt   rE   r.   	PAGE_SIZE)r   r#   r   r   r   r      s    zIBD2SQL.readc             C   s,   | j j| j}tjd|dd d | _|S )Nz>L   r   r   )r.   r    r/   r,   r-   )r   r#   r   r   r   rt      s    zIBD2SQL._read_pagec             C   s   d S )Nr   )r   r   r   r   rv   %  s    zIBD2SQL._read_page_sharec             C   s    | j j| j}|  jd7  _|S )Nr;   )r.   r    r/   )r   r#   r   r   r   rs   (  s    zIBD2SQL._read_page_add1c             C   s   d S )Nr   )r   r   r   r   ru   -  s    zIBD2SQL._read_page_share_add1)FN)	r%   r&   r'   r   r    rt   rv   rs   ru   r   r   r   r   rp   	  s   
rp   PK_NON_LEAFc             C   sX   t  }|j||| |d x:| j|}|dd dkr6P |j| |j d d }qW |S )N)tableidxidr.   	page_typer(   r)   s     r   r/   )r   
init_indexr    	init_dataZget_all_rows)r.   r/   rz   r|   r{   idxr#   r   r   r   FIND_LEAF_PAGE_FROM_ROOT0  s    

r   wc             C   s   | j }|jd}| j  |dkrht||d  dkrh|d |d  tt||d d  d jd }n|d }tj||d  |j	d	| t
||}|S )
Nz.p0r   
   r+   r;      z
.p00000001z
.p00000000zrotate new file, name:)rb   rj   rM   rE   r   intr   r   renamerG   r   )r$   rR   actionr"   ZfindexZnewfilenameZnewfr   r   r   ROTAED_FILE;  s    
4
r   c       $      C   s  d}d}d|krdnd}	d|kr*|d }
n|j dkr:d}
nd}
d	|krN|d	 nd
}|jd k	rb|jndF}|j}|j}d}d}|jdks|jdkrd}d}|jdkrd}dG}t|d |d |d |d |d d}d|krt|nd }d|krt|d }nj|d d r$t|d d d d d }n>|d d dkrL|jd d d d n|jd d d d }|j	|d |d d d d | d!|krt|d! }nt
||| }|j	|d d"| |j|}d#|kr|d$d% ntjd&t|d# }|jdkr|d'krtjj|| j d(| j |d) d k	r.|d) nd' d*tj  d+ }|j dkrtd,t| d-| j d.| j d/t| d0	 t|d1}td2| ntj}|j	d3 |}t }t|}|j| d|d'kr|n|d4|j|j|j||	 |d d5 d6
 |j dkr|j |_!|r tjj"|d |d  }dH|_#xt$|D ]}|j	d7|j# |j|}|d8d9 d:ks6|d;d$ d<ks6||d$d% krq6|j%| g }|r||j!d7 }|r||j!d7 }x4|D ],} |dkr|j&| |
  |d8 }nd S qW q6W nx|d=k r|j|}|j	d7| |d>kr@|j'd?| d@ P tj(dA|dBdC d }|j%| g }|rz||j!d7 }|r||j!d7 }xP|D ]H} |dkr||j&| |
 7 }|d8 }||krt)||}d}nd S qW qW |d'kr|j*  n|j	dD|j t+t,j-d}|jr dn||_.t/ }!i }"x@t$|jD ]2}#t0t1|#||!||||| ||||
||fdE|"|#< q<W x t$|jD ]}#|"|# j2  q~W x t$|jD ]}#|"|# j  qW d S )INr   rN   TFzlines-terminated-byr#   
z;
zfields-terminated-by,r;   onlywith    r   rc   r"   rA   r>   r?   )r@   r"   rA   r>   r?   rootnorH   r<   rP   ra   indexesrootr   FSEG_FRAG_ARRrb   zROOT PAGEID:leafnozLEAF PAGEID:indexidr)   J   z>Qr_   .rd   _z_sql.sqlz-- LOAD DATA INFILE z INTO TABLE `z`.`z` FIELDS TERMINATED BY z5 OPTIONALLY ENCLOSED BY "'" LINES TERMINATED BY '\n';azSQL filename,zoutput is stdoutPK_LEAFr=   )
rz   r{   r.   r|   replacecompletemultifields_terminateddecoder=   zREAD PAGE ID:r3   r4   s   Er(   s     l    r   z
read page(z) faild, will exitz>Lrx   r   z	PARALLEL:)targetargss           r   )3SQLLIMITOUTPUT_FILESIZEFORCEDELETEDr   r
   r   segrG   r   r    r,   ZpackPARALLELr   r   r   schemarb   getpidprintreprr   sysstdoutr   r   r}   REPLACECOMPLETE_INSERTMULTI_VALUEget_dataget_sqlrI   r/   ranger~   writerF   r-   r   rM   r   ctypesZc_uint32valuer   r   IBD2SQL_WORKERstart)$rz   	file_baseoptr   rR   parserFRAGMENT_FILENAME_PREwrited_sizeZwrited_rowsZusehexenclosed_byr   r   r   r   	HAVE_DATAHAVE_DELETEDPAGE_INDEX_IDr.   inoder   r   Zleaf_page_datar"   r$   r/   r   pg2pagesr   r#   rowsqllockZworkerrn   r   r   r   IBD2SQL_SINGLEH  s    


&> 

*
D.

<
6












2r   c             C   s  d|  dt j  d}d}|j|d |dkrt jj||j d|j |	d d k	rX|	d nd d	|  d
t j  d }t|d}t|d| nt	j
}|j|d t }t|	d |	d |	d |	d |	d d}t|}|j|d|dkr|n|d|j|j|j||	d d d	 |jdkr|j|_t jj|	d |	d  }d|	d  }d}x| |j}||ksj|dkrlP |j|}|jr|d |_ntjd|dd d |_|d d! d"ks|d#d$ d%ks|
|d$d& krސwJW d Q R X |j| g }|r
||jd'7 }|r||jd(7 }x:|D ]2}||j|| 7 }||jkr$t||}d}q$W qJW |dkrr|j  |j|d) d S )*NzPROCESS z (pid:z):r   ZSTARTr_   r   rd   Z_pr   z_sql.sqlr   zSQL filename,zoutput is stdoutrc   r"   rA   r>   r?   )r@   r"   rA   r>   r?   r   rH   r=   )	rz   r{   r.   r|   r   r   r   r   r=   r#   r   l    r;   z>Lrx   r   r3   r4   s   Er(   r)   s     r   FTZFINISH) r   r   rG   r   r   r   rb   r   r   r   r   r   r   r   r}   r   r   r   r   r   r   rI   r   r    r   r,   r-   r~   r   r   r   rM   )pr/   r   rR   r   r   r   rz   r   r   r   r   r   r   Zinfoprer   r"   r$   r   r.   r   r   r#   Zpgidr   r   r   r   r   r     sV    H
&6
6



r   c             C   s   d S )Nr   )rz   r   r   r   rR   r   r   r   r   IBD2SQL_MULTI  s    r   )ry   r   )r   ))r,   rg   rY   r   r   ibd2sql.innodb_page.sdir   ibd2sql.innodb_page.pager   r   ibd2sql.innodb_page.fspr   r   r   Zibd2sql.utils.keyring_filer   Zibd2sql.frm.frm2sdir	   ibd2sql.innodb_page.inoder
   ibd2sql.innodb_page.indexr   ibd2sql.innodb_page.tabler   r   Zmultiprocessingr   r   r   objectr   r1   r2   r\   ro   rp   r   r   r   r   r   r   r   r   r   <module>   s>   F 
'

x1