a
    Qu1b•9  ã                   @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlZd dlmZ d dlmZ d dlmZ d dlZd dlZd dlZd dlm Z  G dd	„ d	ƒZ!d
d„ Z"G dd„ dej#ƒZ#dd„ Z$dd„ Z%e&dk�re%ƒ  dS )é    N)ÚNestedJsonRenderer)ÚDecoder)ÚPyBufrKitError)Úunits)Úfs)Úsync_wrapperc                   @   sH   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zd
d„ Zdd„ Z	dd„ Z
dS )ÚBUFRObservedProfilezY
    For reading and parsing of high-resolution
    BUFR observed sounding profiles.
    c                    sX  t ƒ }|| _t|dƒ�º}z| | ¡ ¡}W n– tyv } z4td| jƒ t|ƒ d | _W Y d }~W d   ƒ d S d }~0  tyÀ } z4td| jƒ t|ƒ d | _W Y d }~W d   ƒ d S d }~0 0 W d   ƒ n1 sÖ0    Y  t	ƒ  
|¡}z|d d d d ‰W nJ t�yN } z0td| jƒ t|ƒ t|ƒ d | _W Y d }~d S d }~0 0 tˆƒdk�rtˆd d	 ‰ ˆd }n²‡fd
d„ttˆƒƒD ƒ}t|ƒdk�rˆ|d  d	 }	ˆ|d  d	 }
t|	ƒt|
ƒk�rè|
‰ t ˆ¡ |d ¡}n|	‰ t ˆ¡ |d ¡}n$ˆ|d  d	 ‰ t ˆ¡ |d ¡}‡ fdd„ttˆ ƒƒD ƒd }‡ fdd„ttˆ ƒƒD ƒ}‡ fdd„ttˆ ƒƒD ƒd }‡ fdd„ttˆ ƒƒD ƒ}t|ƒdk�s²t|ƒdk�r¼d | _d S |d }|d }ˆ | }ˆ | }ˆ | }|  |¡| _|  |¡| _|  |¡\| _| _| jd u �r$d | _d S | j| jd< | j| jd< ˆ | d	 }|  |¡| _d S )NÚrbzIssue reading malformed file: é   é   Úvaluer   zIndex error for é   Úmembersc                    s$   g | ]}d t ˆ |  ¡ ƒv r|‘qS )r   )ÚlistÚkeys©Ú.0Úi)Úvalues© ú+/mrms-tiledata/BUFR-archive/BUFRObserved.pyÚ
<listcomp>B   ó    z0BUFRObservedProfile.__init__.<locals>.<listcomp>c                    s    g | ]}d ˆ | d kr|‘qS )Z301113Úidr   r   ©Úflight_datar   r   r   P   r   c                    s    g | ]}d ˆ | d kr|‘qS )Ú301111r   r   r   r   r   r   r   Q   r   c                    s    g | ]}d ˆ | d kr|‘qS )Ú301114r   r   r   r   r   r   r   R   r   c                    s4   g | ],}d ˆ | d krt ˆ | d ƒdkr|‘qS )Z101000r   r   r   )Úlenr   r   r   r   r   S   r   ÚwmoÚdate)r   ÚfilenameÚopenÚprocessÚreadÚAssertionErrorÚprintÚprofiler   r   ZrenderÚ
IndexErrorr   ÚrangeÚcopyÚdeepcopyÚpopÚ_set_station_infoÚstation_infoÚ_set_wmo_numberÚ
wmo_numberÚ_set_observation_timeÚlaunch_dateÚobs_dateÚ_load_profile_data)Úselfr!   ÚdecoderZinsZbufr_messageÚerrZ	json_dataZballoon_metadataÚidxZfd1Úfd2Ztime_idxZwmo_idxZstn_idxZdat_idxZwmo_infoZlaunch_infor.   Zdata_levelsr   )r   r   r   Ú__init__   sx    "B
zBUFRObservedProfile.__init__c                 C   sD   |t | j ¡ ƒv r| j| S | ¡ dkr.| jS | ¡ dkr@| jS dS )zV
        Used to provide dictionary access
        to sounding profile data. 
        r   ZstnN)r   r'   r   Úlowerr0   r.   )r5   Úkeyr   r   r   Ú__getitem__n   s    
zBUFRObservedProfile.__getitem__c                 C   sú  |d }i }|D ]Ð}|d dkrz|d }|D ]L}|d dkrH|d |d< |d dkr`|d |d< |d d	kr,|d |d
< q,|d dkr|d }|D ]L}|d dkr®|d |d< |d dkrÆ|d |d< |d dkr’|d |d< q’q|d d u�rB|d d u�rBt j |d |d |d
 |d |d d�}	t j |	j|	j|	j|	jd�}
ntd| jƒ d }	d }
|	|
fS |
jdk�sv|
jdk�rš|	jdk�rˆ|
}
n|
t jdd� }
nX|
jdk�s²|
jdk�rÄ|
t jdd� }
n.|	jdk�rÖ|
}
n|	jdk�rò|
t jdd� }
|	|
fS )Nr   r   Z301011Z004001r   ÚyearZ004002ÚmonthZ004003ÚdayZ301013Z004004ÚhourZ004005ÚminuteZ004006Úsecond)r>   r?   r@   rA   rB   )r>   r?   r@   rA   z8Cannot parse file. Invalid time info in BUFR record for r   é   é   r   )Zhoursé   é   é   )	Údatetimer>   r?   r@   rA   r&   r!   rB   Ú	timedelta)r5   Zlaunch_info_blockZ	time_listZdatedictZtlZyyyymmdd_listZyyyymmddZhhmmss_listZhhmmssr2   r3   r   r   r   r1   z   sV    ÿÿz)BUFRObservedProfile._set_observation_timec                 C   sš   d}d}t |d ƒdkr’|d }|D ]l}t |d ƒdkr$|d }|D ]J}t |d ƒdkrlt |d ƒ d	¡}qDt |d ƒd
krDt |d ƒ d¡}qDq$|| S )zd
        Parse the WMO info json payload to get
        and return the station's WMO number
        Z00Z000r   r   r   Z301001Z001001r   r   Z001002r
   )ÚstrÚzfill)r5   Zwmo_info_blockZblock_idZstatn_idr   ÚmemberÚentriesÚentryr   r   r   r/   «   s    $&z#BUFRObservedProfile._set_wmo_numberc                 C   s´   i }t |d ƒdkr°|d }|D ]Ž}t |d ƒdkrŽ|d D ]N}t |d ƒdkrf|d }t |ƒ|d< q<t |d ƒdkr<|d }t |ƒ|d	< q<q t |d ƒd
kr t |d ƒ|d< q |S )z„
        Parse the station info block to get 
        the latitude, longitude, and elevation
        of the radiosonde site
        r   r   r   Z301021Z005001r   ÚlatZ006001ÚlonZ007007Zelev)rK   )r5   Zstation_info_blockZstationZ	info_listÚinfoÚvalrP   rQ   r   r   r   r-   Ã   s    z%BUFRObservedProfile._set_station_infoc                 C   sø  g }g }g }g }g }g }g }g }	|||||||||||	|dœ}
|D ]F}|d d }|D ]0}z|
|d    |d ¡ W qV   Y qVY qV0 qVqBi }tjj|td�}tjj|td�}tjj|td�}tjj|td�}tjj|td�}tjj|td�}tjj|td�}tjj|	td�}	tjj||dk< tjj||dk< tjj||dk< tjj||dk< tjj||dk< tjj||dk< tjj||dk< tjj|	|	dk< | jd d	k�rÂt| jd ƒ|	 }	t| jd
 ƒ| }| ¡ dk�rÞ|d |d< n||d< ||d< tj|j	td�|d< tj
|d dd…< |d |d< |d |d< ||d< ||d< ||d
< |	|d< t t |tj¡t |tj¡¡|d< t|ƒt|ƒ  k�rät|ƒ  k�rät|ƒ  k�rät|ƒ  k�rät|ƒ  k�rät|ƒ  k�rät|	ƒk�sôn J d| j ƒ‚|S )z„
        Load the upper-air level reports into
        arrays for each variable and return
        a dictionary of the data
        )ZPRESSUREzGEOPOTENTIAL HEIGHTZGEOPOTENTIALzTEMPERATURE/AIR TEMPERATUREz TEMPERATURE/DRY-BULB TEMPERATUREzDEWPOINT TEMPERATUREzDEW-POINT TEMPERATUREzWIND DIRECTIONzWIND DIRECTION (DEGREE TRUE)z
WIND SPEEDz&LONGITUDE DISPLACEMENT (HIGH ACCURACY)z%LATITUDE DISPLACEMENT (HIGH ACCURACY)r   r   Údescriptionr   )ÚdtypeNrQ   ÚNonerP   iÐ  g      Y@ÚpresZgphtÚhghtgfffffq@ZtmpcZdwpcÚwdirÚwspdZrelhz'Something went wrong parsing the file: )ÚappendÚnpÚmaÚmasked_arrayÚfloatÚmaskedr.   ÚmaxÚzerosÚshapeÚnanÚmpcalcZrelative_humidity_from_dewpointr   ZQuantityZdegKr   r!   )r5   Zlist_of_messagesrW   rX   ZtmpkZdwpkrY   rZ   rP   rQ   ZdatakeyÚmessageZ
level_dataZlevr'   r   r   r   r4   à   s�    ô$6ÿ
ÿ
ÿ
ÿþz&BUFRObservedProfile._load_profile_datac                 C   sh   | j }| j}|d }|d= t |¡}|d |d< |j|d< |j|d< |j|d< |j|d< tj	 
|¡}|S )Nr    r   ÚWMOr>   r?   r@   rA   )r'   r.   ÚpdZ	DataFramer>   r?   r@   rA   ÚpaZTableZfrom_pandas)r'   ZprofdictZstndictr    ÚdfÚtabler   r   r   Úto_apache_tbl4  s    




z!BUFRObservedProfile.to_apache_tblN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r:   r=   r1   r/   r-   r4   rl   r   r   r   r   r      s   S1Tr   c                  C   sr   d  tjd  d¡dd… ¡} tj| dd� ttjd dƒ�"}| tjj	 
¡ ¡ W d  ƒ n1 sd0    Y  dS )z,
    Read from stdin and write to disk.
    ú/r   NéÿÿÿÿT)Úexist_okÚwb)ÚjoinÚsysÚargvÚsplitÚosÚmakedirsr"   ÚwriteÚstdinÚbufferr$   )ÚpathZoutput_filer   r   r   Úwrite_from_pqactF  s    r   c                       s&   e Zd Zd‡ fdd„	ZeeƒZ‡  ZS )ÚGCSFileSystemÚprojectPrivateÚbucketOwnerFullControlc                    s   t ƒ  |||¡S )N)ÚsuperÚ_mkdir)r5   ÚbucketÚaclÚdefault_aclÚkwargs©Ú	__class__r   r   r„   U  s    zGCSFileSystem._mkdir)r�   r‚   )rm   rn   ro   r„   r   ÚmkdirÚ__classcell__r   r   r‰   r   r€   S  s   r€   c           	      C   sº   t ddd�}g d¢}t t dt ¡ fdt ¡ fdt ¡ fdt ¡ fd	t ¡ fg¡¡}d
}d}d }t| d ƒdk rŠt 	| d ¡ 
¡ rŠ|}n|}t ¡ jdd�}tj|||d|d|d� d S )Nzsharp-328923zE/home/ldm/.config/gcloud/legacy_credentials/wxbyte@gmail.com/adc.json)ÚprojectÚtoken)	zSite IDzWMO IDz	Site NameÚStateÚCountryÚLatitudeÚ	LongitudeÚ	ElevationÚFlagr>   r?   r@   rA   rg   zglobal-radiosondes/hires-sonde/zglobal-radiosondes/lores-sonde/rQ   éÈ   ÚSNAPPY)ÚcompressionÚparquetÚoverwrite_or_ignore)Ú
filesystemÚformatÚpartitioningÚexisting_data_behaviorÚfile_options)r€   Údsrœ   ri   ÚschemaÚint32Ústringr   r\   ÚisnanÚallÚParquetFileFormatÚmake_write_optionsÚwrite_dataset)	r'   rk   r   Útable_namesÚ	partitionÚ
hires_pathÚ
lores_pathÚbucket_pathÚpq_format_optsr   r   r   Úwrite_apache_arrowY  s(    




û"þr®   c                  C   s<   t jd } tƒ  t| ƒ}|jd u r&d S | ¡ }t||ƒ dS )Nr   )rv   rw   r   r   r'   rl   r®   )Zoutfilenamer'   Útblr   r   r   Úmainy  s    


r°   Ú__main__)'rv   ry   ÚshutilZncepbufrr*   Únumpyr\   Zpybufrkit.rendererr   Zpybufrkit.decoderr   Zpybufrkit.errorsr   Zmetpy.unitsr   Z
metpy.calcZcalcre   Úpyarrowri   r   Úpyarrow.datasetÚdatasetrŸ   Úpyarrow.parquetr˜   ÚpqZpandasrh   rI   ÚgcsfsÚfsspec.asynr   r   r   r€   r®   r°   rm   r   r   r   r   Ú<module>   s4     2 

