
    g/                        d dl Z d dlZd dlZd dlZddlmZmZmZm	Z	 ddl
mZ  G d d          Z G d de          Z ej                    Zd	d
de dfD ]#ae j                            t                     r n$dad Z G d de          Zd Z G d de          Z G d de          Z G d de          Z G d de          Z G d de          ZddZdS )     N   )
fvecs_read
ivecs_read
bvecs_mmap
fvecs_mmap)knnc                   P    e Zd ZdZd Zd ZddZd Zdd	Zdd
Z	ddZ
d Zd ZdS )Datasetz+ Generic abstract class for a test dataset c                 L    d| _         d| _        d| _        d| _        d| _        dS )z2 the constructor should set the following fields: L2Ndmetricnqnbntselfs    T/var/www/html/mpstechhub/venv/lib/python3.11/site-packages/faiss/contrib/datasets.py__init__zDataset.__init__   s)        c                     t                      )z' return the queries as a (nq, d) array NotImplementedErrorr   s    r   get_querieszDataset.get_queries       !###r   Nc                     t                      )z' return the queries as a (nt, d) array r   r   maxtrains     r   	get_trainzDataset.get_train   r   r   c                     t                      )z' return the queries as a (nb, d) array r   r   s    r   get_databasezDataset.get_database"   r   r      r   r   c              #      K   |                                  }|\  }}| j        |z  |z  | j        |dz   z  |z  }}t          |||          D ]}||t          ||z   |                   V   dS )a7  returns an iterator on database vectors.
        bs is the number of vectors per batch
        split = (nsplit, rank) means the dataset is split in nsplit
        shards and we want shard number rank
        The default implementation just iterates over the full matrix
        returned by get_dataset.
        r   N)r#   r   rangemin	r   bssplitxbnsplitranki0i1j0s	            r   database_iteratorzDataset.database_iterator&   s         46)47dQh+?6+IBB## 	+ 	+BRR"Wb)))*****	+ 	+r   c                     t                      )z7 return the ground truth for k-nearest neighbor search r   r   ks     r   get_groundtruthzDataset.get_groundtruth4   r   r   c                     t                      )z* return the ground truth for range search r   )r   threshs     r   get_groundtruth_rangezDataset.get_groundtruth_range8   r   r   c           
      V    d| j          d| j         d| j         d| j         d| j         
S )Nzdataset in dimension z, with metric z
, size: Q z B z T r   r   s    r   __str__zDataset.__str__<   sU    = = =dk = =7= ='+w= =377= = 	>r   c                    |                                  j        | j        | j        fk    sJ | j        dk    r:|                     d          }|j        d| j        fk    sJ d|j                    |                                 j        | j        | j        fk    sJ |                     d          j        | j        dfk    sJ dS )z8 runs the previous and checks the sizes of the matrices r   {   )r    zshape=   )r5   N)	r   shaper   r   r   r!   r#   r   r6   )r   xts     r   check_sizeszDataset.check_sizes@   s    !!'DGTV+<<<<<7Q;;--B8TV},,,,BHH.G,,,  ""(TWdf,=====##b#))/DGR=@@@@@@r   Nr$   r%   )__name__
__module____qualname____doc__r   r   r!   r#   r2   r6   r9   r;   rA    r   r   r
   r
      s        55  $ $ $$ $ $ $$ $ $+ + + +$ $ $ $$ $ $ $> > >A A A A Ar   r
   c                   6    e Zd ZdZddZd ZddZd Zdd
ZdS )SyntheticDatasetzOA dataset that is not completely random but still challenging to
    index
    r   :  c                 N   t                               |            ||||f\  | _        | _        | _        | _        d}||z   |z   }t          j                            |          }	|		                    ||f          }
t          j
        |
|	                    ||                    }
|
|	                    |          dz  dz   z  }
t          j        |
          }
|
                    d          }
|| _        |
d |         | _        |
|||z            | _        |
||z   d          | _        d S )N
   )size   g?float32)r
   r   r   r   r   r   nprandomRandomStatenormaldotrandsinastyper   r@   r,   xq)r   r   r   r   r   r   seedd1nrsxs              r   r   zSyntheticDataset.__init__O   s   ,-r2rM)$'GbLY""4((IIAr7I##F1bggb!nn%% a#%&F1IIHHYCRC&BrBwJ-BGHH+r   c                     | j         S rB   )rY   r   s    r   r   zSyntheticDataset.get_queriesa   	    wr   Nc                 6    ||n| j         }| j        d |         S rB   )r   r@   r   s     r   r!   zSyntheticDataset.get_traind   s#    '388wyy!!r   c                     | j         S rB   )r,   r   s    r   r#   zSyntheticDataset.get_databaseh   r`   r   d   c                     t          | j        | j        || j        dk    rt          j        nt          j                  d         S )Nr   r   )r   rY   r,   r   faiss	METRIC_L2METRIC_INNER_PRODUCTr4   s     r   r6   z SyntheticDataset.get_groundtruthk   sA    GTWa#{d22EOO8R
 
  	r   )r   rK   rB   )rc   	rD   rE   rF   rG   r   r   r!   r#   r6   rH   r   r   rJ   rJ   J   sx            $  " " " "       r   rJ   z/datasets01/simsearch/041218/z7/mnt/vol/gfsai-flash3-east/ai-group/datasets/simsearch/z/home/z/simsearch/data/zdata/c                 
    | a d S rB   )dataset_basedir)paths    r   set_dataset_basedirrl      s    OOOr   c                   4    e Zd ZdZd Zd ZddZd ZddZdS )	DatasetSIFT1M_
    The original dataset is available at: http://corpus-texmex.irisa.fr/
    (ANN_SIFT1M)
    c                     t                               |            d\  | _        | _        | _        | _        t          dz   | _        d S )N)r$   順 @B '  zsift1M/r
   r   r   r   r   r   rj   basedirr   s    r   r   zDatasetSIFT1M.__init__   <    ,G)$'&2r   c                 0    t          | j        dz             S )Nzsift_query.fvecsr   ru   r   s    r   r   zDatasetSIFT1M.get_queries       $,);;<<<r   Nc                 V    ||n| j         }t          | j        dz             d |         S )Nzsift_learn.fvecsr   r   ru   r   s     r   r!   zDatasetSIFT1M.get_train   1    '388$,);;<<YhYGGr   c                 0    t          | j        dz             S )Nzsift_base.fvecsrx   r   s    r   r#   zDatasetSIFT1M.get_database       $,)::;;;r   c                 d    t          | j        dz             }||dk    sJ |d d d |f         }|S )Nzsift_groundtruth.ivecsrc   r   ru   r   r5   gts      r   r6   zDatasetSIFT1M.get_groundtruth   C    '??@@=8888AAArrEB	r   rB   rh   rH   r   r   rn   rn      w         
3 3 3
= = =H H H H< < <     r   rn   c                 .    t          j        | d          S )NrP   dtype)rQ   ascontiguousarray)r^   s    r   sanitizer      s    3333r   c                   >    e Zd ZdZddZd ZddZddZd ZddZ	dS )DatasetBigANNz_
    The original dataset is available at: http://corpus-texmex.irisa.fr/
    (ANN_SIFT1B)
      c                     t                               |            |dv sJ || _        |dz  }dd|df\  | _        | _        | _        | _        t          dz   | _        d S )N)
r         rM      2   rc      i  r   rr   r$    rs   zbigann/)	r
   r   nb_Mr   r   r   r   rj   ru   )r   r   r   s      r   r   zDatasetBigANN.__init__   sg    AAAAA	E\,/E,A)$'&2r   c                 Z    t          t          | j        dz             d d                    S )Nzbigann_query.bvecs)r   r   ru   r   s    r   r   zDatasetBigANN.get_queries   s)    
4<2F#FGGJKKKr   Nc                 p    ||n| j         }t          t          | j        dz             d |                   S )Nzbigann_learn.bvecs)r   r   r   ru   r   s     r   r!   zDatasetBigANN.get_train   s9    '388
4<2F#FGG		RSSSr   c                 t    t          | j        d| j        z  z             }||dk    sJ |d d d |f         }|S )Nzgnd/idx_%dM.ivecsrc   )r   ru   r   r   s      r   r6   zDatasetBigANN.get_groundtruth   sJ    ':TY'FFGG=8888AAArrEB	r   c                     | j         dk     s
J d            t          t          | j        dz             d | j                           S )Nrc   dataset too large, use iteratorbigann_base.bvecs)r   r   r   ru   r   r   s    r   r#   zDatasetBigANN.get_database   sA    y3 A
4<2E#EFFxxPQQQr   r$   r%   c           	   #      K   t          | j        dz             }|\  }}| j        |z  |z  | j        |dz   z  |z  }}t          |||          D ],}t	          ||t          ||z   |                             V  -d S )Nr   r   )r   ru   r   r'   r   r(   r)   s	            r   r2   zDatasetBigANN.database_iterator   s      '::;;46)47dQh+?6+IBB## 	5 	5B2b#b2gr"2"223444444	5 	5r   )r   rB   rC   
rD   rE   rF   rG   r   r   r!   r6   r#   r2   rH   r   r   r   r      s         
3 3 3 3L L LT T T T   R R R5 5 5 5 5 5r   r   c                   >    e Zd ZdZddZd ZddZddZd ZddZ	dS )DatasetDeep1Bzv
    See
    https://github.com/facebookresearch/faiss/tree/main/benchs#getting-deep1b
    on how to get the data
     ʚ;c                     t                               |            dddddd}||v sJ dd|d	f\  | _        | _        | _        | _        t          d
z   | _        | j        d|| j                 d| _        d S )N100k1M10M100M1B)rq   rr   i r   r   `   i]rs   zdeep1b/deepz_groundtruth.ivecs)	r
   r   r   r   r   r   rj   ru   gt_fname)r   r   
nb_to_names      r   r   zDatasetDeep1B.__init__   s    
 

 Z,.	2u,D)$'&2LLL*TW---/r   c                 J    t          t          | j        dz                       S )Nzdeep1B_queries.fvecs)r   r   ru   r   s    r   r   zDatasetDeep1B.get_queries   s     
4<2H#HIIJJJr   Nc                 p    ||n| j         }t          t          | j        dz             d |                   S )Nzlearn.fvecs)r   r   r   ru   r   s     r   r!   zDatasetDeep1B.get_train   s8    '388
4<-#?@@(KLLLr   c                 ^    t          | j                  }||dk    sJ |d d d |f         }|S )Nrc   )r   r   r   s      r   r6   zDatasetDeep1B.get_groundtruth   s=    &&=8888AAArrEB	r   c                     | j         dk    s
J d            t          t          | j        dz             d | j                            S )Nr   r   
base.fvecs)r   r   r   ru   r   s    r   r#   zDatasetDeep1B.get_database   sF    w%!B
4<,#>??IJJJr   r$   r%   c           	   #      K   t          | j        dz             }|\  }}| j        |z  |z  | j        |dz   z  |z  }}t          |||          D ],}t	          ||t          ||z   |                             V  -d S )Nr   r   )r   ru   r   r'   r   r(   r)   s	            r   r2   zDatasetDeep1B.database_iterator   s      |34446)47dQh+?6+IBB## 	5 	5B2b#b2gr"2"223444444	5 	5r   )r   rB   rC   r   rH   r   r   r   r      s         / / / /K K KM M M M   K K K5 5 5 5 5 5r   r   c                   .    e Zd ZdZddZd Zd Zd	dZdS )
DatasetGlovezD
    Data from http://ann-benchmarks.com/glove-100-angular.hdf5
    NFc                    dd l }|r
J d            |s
t          dz   }|                    |d          | _        d| _        d\  | _        | _        | j        d         j        d         | _        | j        d         j        d         | _	        d S )	Nr   znot implementedzglove/glove-100-angular.hdf5rIP)rc   r   traintest)
h5pyrj   File
glove_h5pyr   r   r   r?   r   r   )r   locdownloadr   s       r   r   zDatasetGlove.__init__  s    ...... 	C!$BBC))C-- /'*03/&)/2r   c                 l    t          j        | j        d                   }t          j        |           |S )Nr   rQ   arrayr   re   normalize_L2r   rY   s     r   r   zDatasetGlove.get_queries  s.    Xdof-..2	r   c                 l    t          j        | j        d                   }t          j        |           |S )Nr   r   r   r,   s     r   r#   zDatasetGlove.get_database  s.    Xdog.//2	r   c                 P    | j         d         }||dk    sJ |d d d |f         }|S )N	neighborsrc   )r   r   s      r   r6   zDatasetGlove.get_groundtruth  s:    _[)=8888AAArrEB	r   )NFrB   rD   rE   rF   rG   r   r   r#   r6   rH   r   r   r   r     sd         
3 
3 
3 
3  
  
     r   r   c                   ,    e Zd ZdZd Zd Zd ZddZdS )DatasetMusic100zO
    get dataset from
    https://github.com/stanis-morozov/ip-nsw#dataset
    c                     t                               |            d\  | _        | _        | _        | _        d| _        t          dz   | _        d S )N)rc   r   rr   rs   r   z
music-100/)	r
   r   r   r   r   r   r   rj   ru   r   s    r   r   zDatasetMusic100.__init__,  sC    ,@)$'&5r   c                 n    t          j        | j        dz   d          }|                    dd          }|S )Nzquery_music100.binrP   r   r   rc   rQ   fromfileru   reshaper   s     r   r   zDatasetMusic100.get_queries2  s6    [(<<INNNZZC  	r   c                 n    t          j        | j        dz   d          }|                    dd          }|S )Nzdatabase_music100.binrP   r   r   rc   r   r   s     r   r#   zDatasetMusic100.get_database7  s6    [(??yQQQZZC  	r   Nc                 n    t          j        | j        dz             }||dk    sJ |d d d |f         }|S )Nzgt.npyrc   )rQ   loadru   r   s      r   r6   zDatasetMusic100.get_groundtruth<  sD    WT\H,--=8888AAArrEB	r   rB   r   rH   r   r   r   r   &  s_         
6 6 6  
  
     r   r   c                   4    e Zd ZdZd Zd ZddZd ZddZdS )	DatasetGIST1Mro   c                     t                               |            d\  | _        | _        | _        | _        t          dz   | _        d S )N)i  rq   rr   rs   zgist1M/rt   r   s    r   r   zDatasetGIST1M.__init__I  rv   r   c                 0    t          | j        dz             S )Nzgist_query.fvecsrx   r   s    r   r   zDatasetGIST1M.get_queriesN  ry   r   Nc                 V    ||n| j         }t          | j        dz             d |         S )Nzgist_learn.fvecsr{   r   s     r   r!   zDatasetGIST1M.get_trainQ  r|   r   c                 0    t          | j        dz             S )Nzgist_base.fvecsrx   r   s    r   r#   zDatasetGIST1M.get_databaseU  r~   r   c                 d    t          | j        dz             }||dk    sJ |d d d |f         }|S )Nzgist_groundtruth.ivecsrc   r   r   s      r   r6   zDatasetGIST1M.get_groundtruthX  r   r   rB   rh   rH   r   r   r   r   C  r   r   r   deep1MFc                 x   | dk    rt                      S | dk    rt                      S |                     d          r/| dk    rdnt          | dd                   }t	          |          S |                     d	          r~| d
d         }|d         dk    rdt          |dd                   z  }n=|dk    rd}n4|d         dk    rdt          |dd                   z  }nJ d|z               t          |          S | dk    rt                      S | dk    rt          |          S t          d| z             )z converts a string describing a dataset to a Dataset object
    Supports sift1M, bigann1M..bigann1B, deep1M..deep1B, music-100 and glove
    sift1Mgist1Mbigannbigann1Br      r   )r   r   rO   NMrr   r   r   r5   Fzdid not recognize suffix )r   z	music-100glove)r   zunknown dataset )	rn   r   
startswithintr   r   r   r   RuntimeError)datasetr   dbsizeszsufs       r   dataset_from_namer   `  sk   
 (	H					H	%	% 9 J..C"4F4F&))))			F	#	# 99s5":.FFd]]FF2Y#Ccrc
OO+FF=5====''''	K		   	G		X.... -7888r   )r   F)osnumpyrQ   re   getpassvecs_ior   r   r   r   exhaustive_searchr   r
   rJ   getuserusernamerj   rk   existsrl   rn   r   r   r   r   r   r   r   rH   r   r   <module>r      s_   
			       D C C C C C C C C C C C " " " " " "8A 8A 8A 8A 8A 8A 8A 8Av% % % % %w % % %\ 7? 	(A++++-  O 
w~~o&&  O  
    G   :4 4 4%5 %5 %5 %5 %5G %5 %5 %5P-5 -5 -5 -5 -5G -5 -5 -5`         7      F    g   :    G   :#9 #9 #9 #9 #9 #9r   