Mercurial > hg > Game > Cerium
changeset 1914:08e9e416c2e0 draft
Implement divide read in regex_mas
author | Masataka Kohagura <e085726@ie.u-ryukyu.ac.jp> |
---|---|
date | Mon, 20 Jan 2014 19:02:22 +0900 |
parents | 53c074e60b08 |
children | effb5653fd5c 68ee760e1c7e |
files | example/regex_mas/Func.h example/regex_mas/Makefile.def example/regex_mas/WordCount.h example/regex_mas/main.cc example/regex_mas/ppe/Exec.cc example/regex_mas/ppe/Print.cc example/regex_mas/ppe/Read.cc example/regex_mas/task_init.cc |
diffstat | 8 files changed, 174 insertions(+), 173 deletions(-) [+] |
line wrap: on
line diff
--- a/example/regex_mas/Func.h Mon Jan 20 16:10:56 2014 +0900 +++ b/example/regex_mas/Func.h Mon Jan 20 19:02:22 2014 +0900 @@ -1,11 +1,11 @@ enum { #include "SysTasks.h" - Task_exec, - Task_read, - Task_print, - RUN_READ, + READ_TASK, RUN_READ_BLOCKS, + MMAP, + TASK_EXEC, RUN_TASK_BLOCKS, + PRINT_TASK, }; #define DATA_NUM 12
--- a/example/regex_mas/Makefile.def Mon Jan 20 16:10:56 2014 +0900 +++ b/example/regex_mas/Makefile.def Mon Jan 20 19:02:22 2014 +0900 @@ -9,10 +9,10 @@ CERIUM = ../../../Cerium -OPT = -g3 -O0 +OPT = -g -O0 CC = clang++ -CFLAGS = -Wall $(OPT) +CFLAGS = -m64 -Wall $(OPT) INCLUDE = -I${CERIUM}/include/TaskManager -I. -I.. LIBS = -L${CERIUM}/TaskManager
--- a/example/regex_mas/WordCount.h Mon Jan 20 16:10:56 2014 +0900 +++ b/example/regex_mas/WordCount.h Mon Jan 20 19:02:22 2014 +0900 @@ -1,10 +1,24 @@ typedef struct wordCount { struct wordCount *self; + /* read task variable */ + int fd; + int read_division_size; + int read_task_number; + int read_task_num; + int read_left_task_num; + int read_filesize; + int read_left_size; + int read_task_blocks; + char *read_text; + CPU_TYPE read_cpu; + + int size; // remaining file size int division_size; // for each word count task int division_out_size; int out_size; + int out_size_; int task_num; // remaining task count int task_blocks; // spawn task one at a time int status_num; @@ -13,12 +27,16 @@ int out_task_num; char *file_mmap; int file_size; - HTaskPtr t_print; + /*set input Data variables*/ int last_task_size; unsigned char *search_word; int *BMskip_table; int search_word_len; int extra_len; -} WordCount; + HTaskPtr t_print; + HTaskPtr t_exec; + +} WordCount, *WordCountPtr; +
--- a/example/regex_mas/main.cc Mon Jan 20 16:10:56 2014 +0900 +++ b/example/regex_mas/main.cc Mon Jan 20 19:02:22 2014 +0900 @@ -29,7 +29,9 @@ int div_read = 0; int array_task_num = 8; int spe_num = 1; +int divide_read_flag = 0; int DIVISION_SIZE = 4*4096; +int READ_DIVISION_SIZE = 4*4096; unsigned char *sword = 0; int task_count = 0; @@ -37,6 +39,7 @@ static int division = 16; // in Kbyte //static unsigned char* search_word; CPU_TYPE spe_cpu = SPE_ANY; +CPU_TYPE read_spe_cpu = SPE_ANY; const char *usr_help_str = "Usage: ./word_count [-a -c -s] [-cpu spe_num] [-sw search_word] [-file filename]\n Required filename & search_word\n"; static double @@ -52,8 +55,7 @@ } st_mmap_t; typedef struct fileread { - struct fileread *self; - char *filename; + struct filesize *self; long fd; long division_size; long task_number; @@ -61,7 +63,6 @@ long filesize; long left_size; int task_blocks; - int task_block_num; char *read_text; CPU_TYPE cpu; } Fileread, *FilereadPtr; @@ -75,121 +76,77 @@ return size; } -SchedDefineTask1(RUN_READ_BLOCKS,read_run16); +SchedDefineTask1(RUN_READ_BLOCKS,read_run); static int -read_run16(SchedTask *manager, void *in, void *out) { +read_run(SchedTask *manager, void *in, void *out) { - FilereadPtr fr = (FilereadPtr)in; + manager->printf("[in divide read mode]\n"); + WordCountPtr w = (WordCountPtr)in; + char* read_text = (char*)out; HTaskPtr wait; - for (int j = 0; j < fr->task_block_num; j++) { + for (int i = 0; (w->read_left_size > 0) && (i < w->read_task_blocks); i++) { + HTaskPtr read = manager->create_task(READ_TASK); + read->set_cpu(w->read_cpu); + + if (i == w->read_task_blocks / 2) wait = read; - for (int i = 0; (fr->left_size > 0) && (i < fr->task_blocks); i++) { - HTaskPtr read = manager->create_task(Task_read); - read->set_cpu(fr->cpu); + read->set_param(0,(long)w->read_task_number); //生成するTaskが何番目か + read->set_param(1,(long)w->read_division_size); //1つのタスクが読み込む量 + if(w->read_left_size <= w->read_division_size){ + read->set_param(2,(long)w->read_left_size); + }else{ + read->set_param(2,(long)w->read_division_size); + } + read->set_param(3,(long)w->fd); //fdの番号の受け渡し - if (i == fr->task_blocks / 2) wait = read; + read->set_outData(0,read_text + w->read_task_number*w->read_division_size, w->read_division_size); - read->set_param(0,(long)fr->task_number); //生成するTaskが何番目か - read->set_param(1,(long)fr->division_size); //1つのタスクが読み込む量 - if(fr->left_size <= fr->division_size){ - read->set_param(2,(long)fr->left_size); - }else{ - read->set_param(2,(long)fr->division_size); - } - read->set_param(3,(long)fr->fd); //fdの番号の受け渡し + w->t_print->wait_for(w->t_exec); + w->t_exec->wait_for(read); + read->spawn(); + + w->read_left_size -= w->read_division_size; + w->read_task_number++; + } - read->set_outData(0,fr->read_text + fr->division_size*fr->task_number, fr->division_size); - read->spawn(); + if (w->read_left_size > 0) { + + HTaskPtr next = manager->create_task(RUN_READ_BLOCKS, (memaddr)&w->self, sizeof(memaddr),read_text,w->read_filesize); - fr->left_size -= fr->division_size; - fr->task_number++; - } - - if (fr->left_size > 0) { - HTaskPtr next = manager->create_task(RUN_READ_BLOCKS, (memaddr)&fr->self, sizeof(memaddr),0,0); - next->wait_for(wait); - next->spawn(); - } + w->t_print->wait_for(next); + w->t_exec->wait_for(next); + next->wait_for(wait); + next->spawn(); } return 0; } -SchedDefineTask1(RUN_READ,divide_read); +SchedDefineTask1(MMAP,my_mmap); static int -divide_read(SchedTask *manager, void *in, void *out) +my_mmap(SchedTask *s, void *in, void *out) { - FilereadPtr fr = (FilereadPtr)in; - char *filename = fr->filename; - - st_mmap_t st_mmap; - long fd = (long)manager->allocate(sizeof(long)); - struct stat sb; + s->printf("[in mmap mode]\n"); - if ((fd=open(filename,O_RDONLY,0666))==0) { - fprintf(stderr,"can't open %s\n",filename); - } - - if (fstat(fd,&sb)) { - fprintf(stderr,"can't fstat %s\n",filename); - } - - st_mmap.size = fix_byte(sb.st_size,division * 1024); - st_mmap.file_mmap = (char*)manager->allocate(st_mmap.size); + WordCountPtr w = (WordCountPtr)in; + /*マッピングだよ!*/ - fr->self = fr; - fr->fd = fd; - fr->cpu = spe_cpu; - fr->task_blocks = 16; - fr->task_number = 0; - fr->filesize = sb.st_size; - fr->left_size = fr->filesize; - fr->division_size = 1024 * division; - fr->task_num = fr->filesize / (1024 * division); - fr->task_num += ((fr->filesize % 1024 * division) != 0); - fr->task_block_num = fr->task_num / fr->task_blocks; - fr->task_block_num += ((fr->task_num % fr->task_blocks) != 0); - - HTaskPtr run = manager->create_task(RUN_READ_BLOCKS, (memaddr)&fr->self, sizeof(memaddr),0,0); - run->spawn(); - - st_mmap.file_mmap = fr->read_text; - - return 0; -} + int map = MAP_PRIVATE; + st_mmap_t st_mmap; -static st_mmap_t -my_read(TaskManager *manager,char *filename) -{ - /*マッピングだよ!*/ - int fd = -1; - st_mmap_t st_mmap; - struct stat sb; - - if ((fd=open(filename,O_RDONLY,0666))==0) { - fprintf(stderr,"can't open %s\n",filename); - } - - if (fstat(fd,&sb)) { - fprintf(stderr,"can't fstat %s\n",filename); - } - - /*sizeをページングサイズの倍数にあわせる*/ - //st_mmap.size = fix_byte(sb.st_size,4096); - st_mmap.size = fix_byte(sb.st_size,division * 1024); - st_mmap.file_mmap = (char*)manager->allocate(st_mmap.size); - - read(fd,st_mmap.file_mmap,st_mmap.size); - + int fd = w->fd; + st_mmap.size = fix_byte(w->read_filesize,4096); + w->file_mmap = (char*)mmap(NULL,st_mmap.size,PROT_READ,map,fd,(off_t)0); if (st_mmap.file_mmap == (caddr_t)-1) { fprintf(stderr,"Can't mmap file\n"); + perror(NULL); exit(0); } - return st_mmap; + return 0; } static void @@ -205,7 +162,7 @@ if (use_task_array) { int task_num = (w->size+size-1)/size; if (task_num>array_task_num) task_num = array_task_num; - task_array = manager->create_task_array(Task_exec,task_num,1,3,1); + task_array = manager->create_task_array(TASK_EXEC,task_num,1,3,1); if (!all) { t_next->wait_for(task_array); } else { @@ -220,7 +177,7 @@ if (w->size < size) size = w->size; if (size==0) break; if (use_task_array) { - t_exec = task_array->next_task_array(Task_exec,t_exec); + t_exec = task_array->next_task_array(TASK_EXEC,t_exec); t_exec->set_inData(0,w->file_mmap + a*w->division_size, size); t_exec->set_inData(1,w->search_word, w->search_word_len); t_exec->set_inData(2,w->BMskip_table, 256); @@ -229,7 +186,7 @@ t_exec->set_outData(0,w->o_data + a*w->out_size, w->division_out_size); } else if (use_compat) { - h_exec = manager->create_task(Task_exec); + h_exec = manager->create_task(TASK_EXEC); h_exec->set_inData(0,w->file_mmap + i*w->division_size, size); h_exec->set_outData(0,w->o_data + i*w->out_size, w->division_out_size); @@ -238,7 +195,7 @@ h_exec->set_cpu(spe_cpu); h_exec->spawn(); } else { - h_exec = manager->create_task(Task_exec, + h_exec = manager->create_task(TASK_EXEC, (memaddr)(w->file_mmap + i*w->division_size), size, (memaddr)(w->o_data + i*w->out_size), w->division_out_size); t_next->wait_for(h_exec); @@ -310,32 +267,49 @@ return skip_table; } + static void run_start(TaskManager *manager, char *filename,unsigned char *search_word, int search_word_len) { - HTaskPtr t_print; - HTaskPtr read_run = NULL; - WordCount *w = (WordCount*)manager->allocate(sizeof(WordCount)); - FilereadPtr fr = (FilereadPtr)manager->allocate(sizeof(FilereadPtr)); - - //fr->self = fr; - fr->self = fr; - fr->filename = filename; + long fd = (long)manager->allocate(sizeof(long)); + struct stat *sb = (struct stat*)manager->allocate(sizeof(struct stat)); + HTaskPtr t_exec = NULL; - st_mmap_t st_mmap; - - if (div_read != 0) { - - read_run = manager->create_task(RUN_READ, (memaddr)&fr->self, sizeof(memaddr),0,0); + if ((fd=open(filename,O_RDONLY,0666))==0) { + fprintf(stderr,"can't open %s\n",filename); + } - printf("in divide read\n"); - - }else { - st_mmap = my_read(manager,filename); - printf("in my read\n"); + if (fstat(fd,sb)) { + fprintf(stderr,"can't fstat %s\n",filename); } - // bzero(w,sizeof(WordCount)); + //WordCount *w = (WordCount*)manager->allocate(sizeof(WordCount)); + WordCountPtr w = (WordCountPtr)manager->allocate(sizeof(WordCount)); + + w->self = w; + w->fd = fd; + w->read_cpu = read_spe_cpu; + w->read_task_blocks = 16; + w->read_filesize = sb->st_size; + w->read_left_size = w->read_filesize; + w->read_division_size = READ_DIVISION_SIZE; + w->read_task_num = w->read_filesize / READ_DIVISION_SIZE; + w->read_task_num += ((w->read_filesize % READ_DIVISION_SIZE) != 0); + w->t_exec = t_exec; + + printf("filesize : %d\n",w->read_filesize); + printf("one_task_size: %d\n",w->read_division_size); + printf("task_num : %d\n",w->read_task_num); + + HTaskPtr r_run = NULL; + + if (divide_read_flag != 0) { + char *read_text = (char*)manager->allocate(w->read_filesize); + r_run = manager->create_task(RUN_READ_BLOCKS, (memaddr)&w->self, sizeof(memaddr),read_text,w->read_filesize); + w->read_text = read_text; + }else { + r_run = manager->create_task(MMAP , (memaddr)&w->self, sizeof(memaddr),0,0); + } /* prepare BMSearch*/ int *skip = (int*)manager->allocate(256 * sizeof(int)); @@ -344,13 +318,21 @@ w->BMskip_table = create_BMskiptable(w->search_word, w->search_word_len, skip); w->extra_len = w->search_word_len - 1; + /* original */ + HTaskPtr t_print; + + //st_mmap_t st_mmap; + //st_mmap = my_mmap(filename); + //w->task_blocks = blocks; w->self = w; w->task_spwaned = 0; /*sizeはdivision_sizeの倍数にしている。*/ - w->size = w->file_size = st_mmap.size; - w->file_mmap = st_mmap.file_mmap; + w->size = w->file_size = w->read_filesize; + //w->file_mmap = st_mmap.file_mmap; + printf("w %lx\n",(long)w); + /* 1task分のデータサイズ(byte) */ if (w->size >= 1024*division) { w->division_size = 1024 * division;/*16kbyte*/ @@ -358,11 +340,12 @@ w->division_size = w->size; } - /* exec output only "match_num" */ - w->status_num = 1; + printf("dvision_size %d\n",w->division_size); + + /* "word num" and "line num" */ + w->status_num = 2; /* taskの数 */ w->task_num = w->size / w->division_size; - printf("task num: %d\n", w->task_num); w->task_num = w->task_num + (w->division_size*w->task_num < w->size); int out_task_num = w->task_num; @@ -373,33 +356,36 @@ } w->out_task_num = out_task_num; + printf("task_num %d\n",w->task_num); + printf("out_task_num %d\n",w->out_task_num); /* out用のdivision_size. statusが2つなので、あわせて16byteになるように、long long(4byte)を使用 */ - w->division_out_size = sizeof(unsigned long long)*2; + w->division_out_size = sizeof(unsigned long long)*4; int out_size = w->division_out_size*out_task_num; w->o_data = (unsigned long long *)manager->allocate(out_size); - w->out_size = 1; + w->out_size_ = out_size; + w->out_size = 4; + printf("out size %d\n",out_size); /*各SPEの結果を合計して出力するタスク*/ - t_print = manager->create_task(Task_print, + t_print = manager->create_task(PRINT_TASK, (memaddr)&w->self,sizeof(memaddr),0,0); w->t_print = t_print; - /* Task を task_blocks ずつ起動する Task */ - /* serialize されていると仮定する... */ - HTaskPtr t_exec = manager->create_task(RUN_TASK_BLOCKS, - (memaddr)&w->self,sizeof(memaddr),0,0); - - if (read_run != NULL) { - t_exec->wait_for(read_run); - read_run->spawn(); + for(int i=0;i<1;i++) { + /* Task を task_blocks ずつ起動する Task */ + /* serialize されていると仮定する... */ + t_exec = manager->create_task(RUN_TASK_BLOCKS, + (memaddr)&w->self,sizeof(memaddr),0,0); + t_exec->wait_for(r_run); + t_print->wait_for(t_exec); + // t_exec->iterate(4); + t_exec->spawn(); } - - t_print->wait_for(t_exec); - t_exec->iterate(2); - + t_print->wait_for(r_run); + r_run->spawn(); t_print->spawn(); } @@ -422,8 +408,6 @@ } else if (strcmp(argv[i], "-c") == 0) { use_task_array = 0; use_compat = 1; - } else if (strcmp(argv[i], "-dr") == 0) { - div_read = 1; } else if (strcmp(argv[i], "-s") == 0) { use_task_array = 0; use_compat = 0; @@ -440,6 +424,8 @@ if (spe_num==0) spe_num = 1; } else if (strcmp(argv[i], "-sw") == 0) { sword = (unsigned char*)argv[i+1]; + } else if (strcmp(argv[i], "-dr") == 0) { + divide_read_flag = 1; } } if ((filename==0) || (sword==0)) {
--- a/example/regex_mas/ppe/Exec.cc Mon Jan 20 16:10:56 2014 +0900 +++ b/example/regex_mas/ppe/Exec.cc Mon Jan 20 19:02:22 2014 +0900 @@ -6,7 +6,7 @@ #define max(a,b)((a)>(b)?a:b) /* これは必須 */ -SchedDefineTask1(Task_exec,task_exec); +SchedDefineTask1(TASK_EXEC,task_exec); //ボイヤームーア法による文字列検索アルゴリズム static int BM_method(unsigned char *text,int text_len, @@ -41,7 +41,7 @@ int sw_len = (int)s->get_inputSize(1); //get_param - long task_count = (long)s->get_param(0); //何番目のtaskか + //long task_count = (long)s->get_param(0); //何番目のtaskか //s->printf("[exec No: %lld]\n",task_count+1); unsigned long long *o_data = (unsigned long long*)s->get_output(wbuf,0);
--- a/example/regex_mas/ppe/Print.cc Mon Jan 20 16:10:56 2014 +0900 +++ b/example/regex_mas/ppe/Print.cc Mon Jan 20 19:02:22 2014 +0900 @@ -5,10 +5,10 @@ #include "WordCount.h" /* これは必須 */ -SchedDefineTask1(Task_print,task_print); +SchedDefineTask1(PRINT_TASK,print_task); static int -task_print(SchedTask *s, void *rbuf, void *wbuf) +print_task(SchedTask *s, void *rbuf, void *wbuf) { WordCount *w = *(WordCount**)rbuf; unsigned long long *idata = w->o_data;
--- a/example/regex_mas/ppe/Read.cc Mon Jan 20 16:10:56 2014 +0900 +++ b/example/regex_mas/ppe/Read.cc Mon Jan 20 19:02:22 2014 +0900 @@ -11,26 +11,22 @@ #include "Func.h" /* これは必須 */ -SchedDefineTask1(Task_read,task_read); +SchedDefineTask1(READ_TASK,read_task); static int -task_read(SchedTask *s, void *rbuf, void *wbuf) +read_task(SchedTask *s, void *rbuf, void *wbuf) { long task_number = (long)s->get_param(0); //何番目のtaskか long division_size = (long)s->get_param(1); long read_size = (long)s->get_param(2); - long fd = (long)s->get_param(3); //fdの番号の受け取り + long fd = (long)s->get_param(3); - char *readtext = (char*)s->get_output(wbuf,0); + char *read_text = (char*)s->get_output(wbuf,0); - //char *text = (char*)s->allocate(sizeof(char)*read_size + 1); - //char text[(long)read_size]; - //text[(long)read_size] = '\0'; - //s->printf("[read No: %lld]\n",task_number+1); + pread(fd, read_text, (long)read_size , division_size*task_number); - pread(fd, readtext, read_size , division_size*task_number); - - //readtext = text; - + //s->printf("[start task No. %d]\n",task_number); + //s->printf("%s\n",read_text); + //s->printf("in divide_read\n"); return 0; }
--- a/example/regex_mas/task_init.cc Mon Jan 20 16:10:56 2014 +0900 +++ b/example/regex_mas/task_init.cc Mon Jan 20 19:02:22 2014 +0900 @@ -1,34 +1,35 @@ #include "Func.h" #include "Scheduler.h" +#ifdef __CERIUM_GPU__ #include "GpuScheduler.h" +#endif /* 必ずこの位置に書いて */ -#ifndef __CERIUM_GPU__ -SchedExternTask(Task_exec); -#endif -SchedExternTask(Task_print); -SchedExternTask(Task_read); -SchedExternTask(RUN_READ); +SchedExternTask(TASK_EXEC); +SchedExternTask(READ_TASK); SchedExternTask(RUN_READ_BLOCKS); +SchedExternTask(MMAP); +SchedExternTask(TASK_EXEC); SchedExternTask(RUN_TASK_BLOCKS); +SchedExternTask(PRINT_TASK); /** * この関数は ../spe/spe-main と違って * 自分で呼び出せばいい関数なので * 好きな関数名でおk (SchedRegisterTask は必須) */ + void task_init(void) { #ifdef __CERIUM_GPU__ - GpuSchedRegister(Task_exec, "gpu/Exec.cl", "run"); -#else - SchedRegister(Task_exec); + GpuSchedRegister(TASK_EXEC, "gpu/Exec.cl", "run"); #endif - SchedRegister(Task_read); - SchedRegister(Task_print); - SchedRegister(RUN_READ); + SchedRegister(TASK_EXEC); + SchedRegister(READ_TASK); SchedRegister(RUN_READ_BLOCKS); + SchedRegister(MMAP); SchedRegister(RUN_TASK_BLOCKS); + SchedRegister(PRINT_TASK); }