text: Search.setIndex({docnames:["Tutorial","cogdata","cogdata.data_savers","cogdata.datasets","cogdata.tasks","cogdata.utils","cogdata.utils.cogview","index"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":2,sphinx:56},filenames:["Tutorial.rst","cogdata.rst","cogdata.data_savers.rst","cogdata.datasets.rst","cogdata.tasks.rst","cogdata.utils.rst","cogdata.utils.cogview.rst","index.rst"],objects:{"cogdata.data_manager":{DataManager:[1,1,1,""]},"cogdata.data_manager.DataManager":{clean:[1,2,1,""],fetch_datasets:[1,2,1,""],fetch_datasets_states:[1,2,1,""],list:[1,2,1,""],load_task:[1,2,1,""],merge:[1,2,1,""],new_dataset:[1,2,1,""],new_task:[1,2,1,""],process:[1,2,1,""],process_single:[1,2,1,""],split:[1,2,1,""]},"cogdata.data_processor":{DataProcessor:[1,1,1,""],initialize_distributed:[1,3,1,""]},"cogdata.data_processor.DataProcessor":{run_monitor:[1,2,1,""],run_single:[1,2,1,""]},"cogdata.data_savers":{base_saver:[2,0,0,"-"],binary_saver:[2,0,0,"-"],tar_saver:[2,0,0,"-"]},"cogdata.data_savers.base_saver":{BaseSaver:[2,1,1,""]},"cogdata.data_savers.base_saver.BaseSaver":{commit:[2,2,1,""],merge:[2,2,1,""],save:[2,2,1,""],split:[2,2,1,""],suffix:[2,4,1,""]},"cogdata.data_savers.binary_saver":{BinarySaver:[2,1,1,""]},"cogdata.data_savers.binary_saver.BinarySaver":{commit:[2,2,1,""],mapping:[2,4,1,""],max_buffer_size:[2,4,1,""],merge:[2,2,1,""],save:[2,2,1,""],split:[2,2,1,""],suffix:[2,4,1,""]},"cogdata.data_savers.tar_saver":{TarSaver:[2,1,1,""]},"cogdata.data_savers.tar_saver.TarSaver":{commit:[2,2,1,""],merge:[2,2,1,""],save:[2,2,1,""],split:[2,2,1,""],suffix:[2,4,1,""]},"cogdata.datasets":{binary_dataset:[3,0,0,"-"],rar_dataset:[3,0,0,"-"],tar_dataset:[3,0,0,"-"],zip_dataset:[3,0,0,"-"]},"cogdata.datasets.binary_dataset":{BinaryDataset:[3,1,1,""]},"cogdata.datasets.binary_dataset.BinaryDataset":{__getitem__:[3,2,1,""],__init__:[3,2,1,""],__len__:[3,2,1,""]},"cogdata.datasets.rar_dataset":{StreamingRarDataset:[3,1,1,""]},"cogdata.datasets.rar_dataset.StreamingRarDataset":{__del__:[3,2,1,""],__init__:[3,2,1,""],__iter__:[3,2,1,""],__len__:[3,2,1,""],__next__:[3,2,1,""]},"cogdata.datasets.tar_dataset":{TarDataset:[3,1,1,""]},"cogdata.datasets.tar_dataset.TarDataset":{__getitem__:[3,2,1,""],__init__:[3,2,1,""],__len__:[3,2,1,""]},"cogdata.datasets.zip_dataset":{ZipDataset:[3,1,1,""]},"cogdata.datasets.zip_dataset.ZipDataset":{__getitem__:[3,2,1,""],__init__:[3,2,1,""],__len__:[3,2,1,""]},"cogdata.tasks":{image_text_tokenization_task:[4,0,0,"-"]},"cogdata.tasks.image_text_tokenization_task":{ImageTextTokenizationTask:[4,1,1,""]},"cogdata.tasks.image_text_tokenization_task.ImageTextTokenizationTask":{__init__:[4,2,1,""],get_transform_fn:[4,2,1,""],process:[4,2,1,""],read_text:[4,2,1,""]},"cogdata.utils":{eprogress:[5,0,0,"-"],helpers:[5,0,0,"-"],logger:[5,0,0,"-"],progress_record:[5,0,0,"-"],register:[5,0,0,"-"]},"cogdata.utils.cogview":{api:[6,0,0,"-"],unified_tokenizer:[6,0,0,"-"]},"cogdata.utils.cogview.api":{code2img:[6,3,1,""],img2code:[6,3,1,""],new_model:[6,3,1,""]},"cogdata.utils.cogview.unified_tokenizer":{get_tokenizer:[6,3,1,""]},"cogdata.utils.eprogress":{LineProgress:[5,1,1,""],MultiProgressManager:[5,1,1,""],ProgressBar:[5,1,1,""]},"cogdata.utils.eprogress.LineProgress":{update:[5,2,1,""]},"cogdata.utils.eprogress.MultiProgressManager":{clear:[5,2,1,""],put:[5,2,1,""],skip_upline:[5,2,1,""],update:[5,2,1,""],update_title:[5,2,1,""]},"cogdata.utils.eprogress.ProgressBar":{filter_str:[5,2,1,""],lock:[5,2,1,""],update:[5,2,1,""]},"cogdata.utils.helpers":{dir_size:[5,3,1,""],format_file_size:[5,3,1,""],get_last_line:[5,3,1,""],get_registered_cls:[5,3,1,""]},"cogdata.utils.logger":{get_logger:[5,3,1,""],set_logger:[5,3,1,""]},"cogdata.utils.progress_record":{ProgressRecord:[5,1,1,""]},"cogdata.utils.progress_record.ProgressRecord":{get_all:[5,2,1,""],update:[5,2,1,""]},"cogdata.utils.register":{register:[5,3,1,""]},cogdata:{data_manager:[1,0,0,"-"],data_processor:[1,0,0,"-"]}},objnames:{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","method","Python method"],"3":["py","function","Python function"],"4":["py","attribute","Python attribute"]},objtypes:{"0":"py:module","1":"py:class","2":"py:method","3":"py:function","4":"py:attribute"},terms:{"100":[0,5],"10737418240":2,"1088":0,"10mb":1,"207":0,"233":1,"23gb":1,"256":0,"4mb":0,"5000":4,"7mb":0,"abstract":[2,5],"byte":[0,3,6],"class":[1,2,3,4,5],"final":4,"float":5,"function":[1,3,4],"int":[1,3,4,5],"long":0,"new":[0,5,6],"return":[1,3,4,5,6],"static":[1,5],"true":[3,5],One:1,The:[1,2,3,4,5,6,7],There:0,Use:[0,4],Used:3,With:7,__del__:3,__getitem__:3,__init__:[3,4],__iter__:3,__len__:3,__next__:3,abc:2,about:0,add:5,addit:[0,1,7],address:1,all:[0,1,2,4,5],all_dataset:1,allow:6,amount:7,api:[1,5,7],arg:[1,2,3,5],argpars:1,args_dict:1,argument:7,arrai:3,avail:1,averag:[0,5],bar:[4,5],base:[1,2,3,4,5,6],base_dir:1,base_sav:[1,7],base_task:4,basesav:2,basetask:4,batch:[4,6],becaus:0,bin:[0,2],binari:[2,3],binary_dataset:[1,7],binary_sav:[1,7],binarydataset:3,binarysav:[0,2],bool:[0,2,3],booltensor:2,both:1,bpe:6,buffer:2,build:0,built:3,bytetensor:2,call:1,can:[0,2,7],chang:6,check:7,chines:6,classmethod:[2,5],clean:[1,7],clear:5,cli:7,clone:6,close:3,cls:5,cmd:1,code2img:6,code:[6,7],cogdata:0,cogdata_config:[0,1],cogdata_info:[0,1],cogdata_task_test_task:0,cogdata_workspac:1,cogview:[0,1,5,7],com:6,commend:[0,1],commit:[2,4],comput:0,config:[1,4],connect:1,consid:1,consol:1,contain:[1,3],content:[5,7],convert:6,crash:0,creat:[1,7],create_dataset:0,create_task:0,create_tim:0,cuda:4,current:[1,3,5],current_dir:1,custom:7,damag:0,dat:2,data:[0,1,2,3,4,5,7],data_fil:0,data_format:0,data_manag:7,data_processor:7,data_sav:[1,7],dataload:[0,1,4],dataloader_num_work:0,datamanag:1,dataprocessor:1,dataset1:1,dataset2:1,dataset:[1,4,5,7],dataset_dir:4,dataset_nam:0,defin:[1,6],depend:6,descript:0,detect:1,detoken:6,dict:[0,1,4],dir:1,dir_siz:5,direct:6,directli:3,directori:[0,5],distribut:1,doe:6,don:0,dtype:[0,2,3],dure:5,effici:7,empti:5,encod:6,end:6,entri:1,eprogress:[1,7],everi:4,examin:0,exampl:0,exist:1,exit:0,expect:0,extens:6,fals:[2,3],fd5061f6:0,featur:7,fetch_dataset:1,fetch_datasets_st:1,file:[0,1,2,3,4,5],file_s:[2,3],filenam:[3,5],files:5,filter:5,filter_str:5,first:[1,5],firstli:0,float32:[0,2],floattensor:2,folder:[0,1,2,4,5],forget:0,format:0,format_file_s:5,from:[1,3,4,6],full_filenam:[2,3],gener:[3,6],get:[1,3,5],get_al:5,get_last_lin:5,get_logg:5,get_registered_cl:5,get_token:6,get_transform_fn:4,git:6,github:6,global:1,going:1,googl:6,gpt:6,gpu:[1,3],group:[0,7],handl:[0,1,4],hang:[0,1],help:0,helper:[1,7],here:0,http:[0,6],human:5,idx:3,imag:[0,3,4,6],image_text_token:1,image_text_tokenization_task:[1,7],imagetexttokenizationtask:[0,4],img2cod:6,img:6,img_siz:[0,4],img_tokenizer_path:6,implement:6,includ:4,index:[0,3],info:0,infolist:0,infom:5,initi:[1,7],initialize_distribut:1,input:2,input_path:2,instal:[6,7],install_unrarlib:0,instanc:[5,6],int32:[0,2,3],int64:[0,2],inttensor:2,item:3,iter:[1,3],json:[0,1,4],json_k:[1,4],just:0,kei:5,kudo:6,kwarg:[2,3,4,5],languag:6,larg:7,last:5,launch:[0,1],length:[0,3],length_per_sampl:[0,3],lenth_per_sampl:3,let:0,line:5,lineprogress:5,list:[0,1,4],littl:7,load:[1,3,6],load_task:1,local:[1,3,5],local_rank:1,lock:5,log:[0,5],log_dir:5,logger:[1,7],longtensor:[2,6],main:[5,7],main_pid_35218:0,main_pid_40494:0,mainli:6,make:6,manag:[1,7],manual:1,map:[2,3],master_addr:1,master_port:1,max_buffer_s:2,mean:0,merg:[1,2,5,7],messag:0,meta_info:0,migrat:1,mmap:3,mode:[0,2,4],model:6,model_path:0,modifi:0,modul:7,monitor:1,move:0,multi:7,multipl:[1,3],multiprocess:1,multiprogressmanag:5,n10148035:0,name:[0,1,2,4,5],namespac:1,need:[0,2,5],network:6,neural:6,new_dataset:1,new_model:6,new_task:1,next:3,none:[0,1,3,4,5,6],normal:5,now:0,nproc:0,number:[0,1,3,5],numpi:3,object:[1,5],one:[0,1,2],onli:1,openai:6,org:0,output:[0,1,2],output_dir:2,output_path:2,overwrit:2,packag:[1,7],pair:6,param:6,paramet:[1,2,3,4,5,6],pars:1,part0:0,part1:0,part2:0,part:1,part_0:0,part_1:0,path:[0,1,2,3,4,5],pending_str:5,per:0,piec:0,pip:[0,6],pointer:[2,3],port:1,possibl:0,postprocess:6,pre:[0,6],predetermin:6,preload:3,pretrain:6,previou:1,prior:6,process:[1,2,3,4,5,7],process_singl:1,process_single_entri:7,processed_s:0,processor:1,progress:[0,1,4,5],progress_bar:5,progress_record:[1,4,7],progressbar:[4,5],progressrecord:5,properti:5,provid:1,pure:6,put:5,pypi:0,python:6,quick:7,random:0,rank:[0,1,3,5],rank_0:0,rank_1:0,rar:[1,3],rar_dataset:[1,7],ratio:0,raw:[0,1,3,6],raw_siz:0,read:[4,5],read_text:4,real:1,realli:1,record:5,regist:[1,7],remov:[0,5],repaint:5,report:4,result:[0,3],root:[1,2,5],run:[1,7],run_monitor:1,run_singl:1,same:[0,6],sampl:[0,2,3,5],save:[2,4],saver:[0,1,4,5],saver_typ:0,select:3,sennrich:6,sentenc:6,sentencepiec:6,set:[1,3,5],set_logg:5,setup:6,should:1,show:0,simpl:0,singlton:6,size:[0,3,5,6],skip:5,skip_uplin:5,some:[0,1],sp_token:[1,5,7],specif:[5,6],speed:[0,5],split:[1,2,3,7],split_merged_fil:0,stai:0,start:7,statu:[1,5],str:[1,2,3,4,5],streamingrardataset:3,string:1,structur:0,sub_dataset:4,subfold:1,subpackag:7,subset:0,subword:6,successfulli:4,sudo:0,suffix:2,sum:5,summari:0,symbol:5,system:6,t_co:3,tar:[0,2],tar_dataset:[1,7],tar_sav:[1,7],tardataset:[0,3],target_path:5,tarsav:2,task:[1,5,7],task_id:[0,1],task_typ:0,taskid:1,tasknam:1,tcp:1,templat:1,tensor:[2,3],termin:0,test:0,test_d:0,test_task:0,text:[0,4,6],text_fil:0,text_format:0,them:0,thi:[0,1,4,6],titl:5,tmp:1,tokeiz:0,token:[0,4,6],too:0,torch:[0,1,2,3,6],torchvis:4,total:[0,3,5],totensor:4,train:6,transform:[3,4],transform_fn:[1,3],translat:5,tsv:4,tupl:[1,3],two:7,txt:4,txt_file:4,txt_len:0,type:[0,1,3,4,5],uint8:[0,2],unified_token:[1,5,7],unigram:6,unit:6,unprocess:[0,1],unsupervis:6,updat:5,update_titl:5,url:0,usag:0,use:[0,1,4,7],used:6,util:[1,3,7],valid:3,veri:7,via:4,vocabulari:6,vqvae:6,vqvae_hard_biggerset_011:0,vqvae_token:[1,5,7],vqvae_zc:[1,5,7],wait:[0,1],where:6,which:[1,2],width:5,world_siz:[1,3],write:2,you:7,zip:[1,3],zip_dataset:[1,7],zipdataset:3},titles:["Quick Start","cogdata","cogdata.data_savers package","cogdata.datasets package","cogdata.tasks package","cogdata.utils package","cogdata.utils.cogview package","Welcome to Cogdata\u2019s documentation!"],titleterms:{api:6,argument:1,base_sav:2,binary_dataset:3,binary_sav:2,check:0,clean:0,cli:1,cogdata:[1,2,3,4,5,6,7],cogview:6,creat:0,data_manag:1,data_processor:1,data_sav:2,dataset:[0,3],document:7,eprogress:5,helper:5,image_text_tokenization_task:4,initi:0,instal:0,logger:5,merg:0,modul:[1,2,3,4,5,6],packag:[2,3,4,5,6],process:0,process_single_entri:1,progress_record:5,quick:0,rar_dataset:3,regist:5,sp_token:6,split:0,start:0,subpackag:[1,5],tar_dataset:3,tar_sav:2,task:[0,4],unified_token:6,util:[5,6],vqvae_token:6,vqvae_zc:6,welcom:7,zip_dataset:3}})