changeset 1914:08e9e416c2e0 draft

Implement divide read in regex_mas
author Masataka Kohagura <e085726@ie.u-ryukyu.ac.jp>
date Mon, 20 Jan 2014 19:02:22 +0900
parents 53c074e60b08
children effb5653fd5c 68ee760e1c7e
files example/regex_mas/Func.h example/regex_mas/Makefile.def example/regex_mas/WordCount.h example/regex_mas/main.cc example/regex_mas/ppe/Exec.cc example/regex_mas/ppe/Print.cc example/regex_mas/ppe/Read.cc example/regex_mas/task_init.cc
diffstat 8 files changed, 174 insertions(+), 173 deletions(-) [+]
line wrap: on
line diff
--- a/example/regex_mas/Func.h	Mon Jan 20 16:10:56 2014 +0900
+++ b/example/regex_mas/Func.h	Mon Jan 20 19:02:22 2014 +0900
@@ -1,11 +1,11 @@
 enum {
 #include "SysTasks.h"
-    Task_exec,
-    Task_read,
-    Task_print,
-    RUN_READ,
+    READ_TASK,
     RUN_READ_BLOCKS,
+    MMAP,
+    TASK_EXEC,
     RUN_TASK_BLOCKS,
+    PRINT_TASK,
 };
 
 #define DATA_NUM 12
--- a/example/regex_mas/Makefile.def	Mon Jan 20 16:10:56 2014 +0900
+++ b/example/regex_mas/Makefile.def	Mon Jan 20 19:02:22 2014 +0900
@@ -9,10 +9,10 @@
 CERIUM = ../../../Cerium
 
 
-OPT =  -g3 -O0
+OPT =  -g -O0
 
 CC      = clang++
-CFLAGS  =  -Wall $(OPT) 
+CFLAGS  = -m64 -Wall $(OPT) 
 
 INCLUDE = -I${CERIUM}/include/TaskManager -I. -I..
 LIBS = -L${CERIUM}/TaskManager
--- a/example/regex_mas/WordCount.h	Mon Jan 20 16:10:56 2014 +0900
+++ b/example/regex_mas/WordCount.h	Mon Jan 20 19:02:22 2014 +0900
@@ -1,10 +1,24 @@
 
 typedef struct wordCount {
     struct wordCount *self;
+    /* read task variable */
+    int fd;
+    int read_division_size;
+    int read_task_number;
+    int read_task_num;
+    int read_left_task_num;
+    int read_filesize;
+    int read_left_size;
+    int read_task_blocks;
+    char *read_text;
+    CPU_TYPE read_cpu;
+
+
     int size;             // remaining file size
     int division_size;    // for each word count task
     int division_out_size;
     int out_size;
+    int out_size_;
     int task_num;         // remaining task count
     int task_blocks;      // spawn task one at a time
     int status_num;
@@ -13,12 +27,16 @@
     int out_task_num;
     char *file_mmap;
     int file_size;
-    HTaskPtr t_print;
+
     /*set input Data variables*/
     int last_task_size;
     unsigned char *search_word;
     int *BMskip_table;
     int search_word_len;
     int extra_len;
-} WordCount;
 
+    HTaskPtr t_print;
+    HTaskPtr t_exec;
+
+} WordCount, *WordCountPtr;
+
--- a/example/regex_mas/main.cc	Mon Jan 20 16:10:56 2014 +0900
+++ b/example/regex_mas/main.cc	Mon Jan 20 19:02:22 2014 +0900
@@ -29,7 +29,9 @@
 int div_read = 0;
 int array_task_num = 8;
 int spe_num = 1;
+int divide_read_flag = 0;
 int DIVISION_SIZE = 4*4096;
+int READ_DIVISION_SIZE = 4*4096;
 
 unsigned char *sword = 0;
 int task_count = 0;
@@ -37,6 +39,7 @@
 static int division = 16; // in Kbyte
 //static unsigned char* search_word;
 CPU_TYPE spe_cpu = SPE_ANY;
+CPU_TYPE read_spe_cpu = SPE_ANY;
 const char *usr_help_str = "Usage: ./word_count [-a -c -s] [-cpu spe_num] [-sw search_word] [-file filename]\n       Required filename & search_word\n";
 
 static double
@@ -52,8 +55,7 @@
 } st_mmap_t;
 
 typedef struct fileread {
-   struct fileread *self;
-   char *filename;
+   struct filesize *self;
    long fd;
    long division_size;
    long task_number;
@@ -61,7 +63,6 @@
    long filesize;
    long left_size;
    int task_blocks;
-   int task_block_num;
    char *read_text;
    CPU_TYPE cpu;
 } Fileread, *FilereadPtr;
@@ -75,121 +76,77 @@
     return size;
 }
 
-SchedDefineTask1(RUN_READ_BLOCKS,read_run16);
+SchedDefineTask1(RUN_READ_BLOCKS,read_run);
 
 static int
-read_run16(SchedTask *manager, void *in, void *out) {
+read_run(SchedTask *manager, void *in, void *out) {
 
-    FilereadPtr fr = (FilereadPtr)in;
+    manager->printf("[in divide read mode]\n");
+    WordCountPtr w = (WordCountPtr)in;
+    char* read_text = (char*)out;
     HTaskPtr wait;
 
-    for (int j = 0; j < fr->task_block_num; j++) {
+    for (int i = 0; (w->read_left_size > 0) && (i < w->read_task_blocks); i++) {
+        HTaskPtr read = manager->create_task(READ_TASK);
+        read->set_cpu(w->read_cpu);
+
+        if (i == w->read_task_blocks / 2) wait = read;
 
-        for (int i = 0; (fr->left_size > 0) && (i < fr->task_blocks); i++) {
-            HTaskPtr read = manager->create_task(Task_read);
-            read->set_cpu(fr->cpu);
+        read->set_param(0,(long)w->read_task_number);  //生成するTaskが何番目か
+        read->set_param(1,(long)w->read_division_size);  //1つのタスクが読み込む量
+        if(w->read_left_size <= w->read_division_size){
+            read->set_param(2,(long)w->read_left_size);
+        }else{
+            read->set_param(2,(long)w->read_division_size);
+        }
+        read->set_param(3,(long)w->fd);  //fdの番号の受け渡し
 
-            if (i == fr->task_blocks / 2) wait = read;
+        read->set_outData(0,read_text + w->read_task_number*w->read_division_size, w->read_division_size);
 
-            read->set_param(0,(long)fr->task_number);  //生成するTaskが何番目か
-            read->set_param(1,(long)fr->division_size);  //1つのタスクが読み込む量
-            if(fr->left_size <= fr->division_size){
-                read->set_param(2,(long)fr->left_size);
-            }else{
-                read->set_param(2,(long)fr->division_size);
-            }
-            read->set_param(3,(long)fr->fd);  //fdの番号の受け渡し
+        w->t_print->wait_for(w->t_exec);
+        w->t_exec->wait_for(read);
+        read->spawn();
+
+        w->read_left_size -= w->read_division_size;
+        w->read_task_number++;
+    }
 
-            read->set_outData(0,fr->read_text + fr->division_size*fr->task_number, fr->division_size);
-            read->spawn();
+    if (w->read_left_size > 0) {
+
+        HTaskPtr next = manager->create_task(RUN_READ_BLOCKS, (memaddr)&w->self, sizeof(memaddr),read_text,w->read_filesize);
 
-            fr->left_size -= fr->division_size;
-            fr->task_number++;
-        }
-
-        if (fr->left_size > 0) {
-            HTaskPtr next = manager->create_task(RUN_READ_BLOCKS, (memaddr)&fr->self, sizeof(memaddr),0,0);
-            next->wait_for(wait);
-            next->spawn();
-        }
+        w->t_print->wait_for(next);
+        w->t_exec->wait_for(next);
+        next->wait_for(wait);
+        next->spawn();
     }
     return 0;
 }
 
-SchedDefineTask1(RUN_READ,divide_read);
+SchedDefineTask1(MMAP,my_mmap);
 
 static int
-divide_read(SchedTask *manager, void *in, void *out)
+my_mmap(SchedTask *s, void *in, void *out)
 {
-    FilereadPtr fr = (FilereadPtr)in;
-    char *filename = fr->filename;
-
-    st_mmap_t st_mmap;
-    long   fd = (long)manager->allocate(sizeof(long));
-    struct stat sb;
+    s->printf("[in mmap mode]\n");
 
-    if ((fd=open(filename,O_RDONLY,0666))==0) {
-        fprintf(stderr,"can't open %s\n",filename);
-    }
-
-    if (fstat(fd,&sb)) {
-        fprintf(stderr,"can't fstat %s\n",filename);
-    }
-
-    st_mmap.size = fix_byte(sb.st_size,division * 1024);
-    st_mmap.file_mmap = (char*)manager->allocate(st_mmap.size);
+    WordCountPtr w = (WordCountPtr)in;
+    /*マッピングだよ!*/
 
-    fr->self = fr;
-    fr->fd = fd;
-    fr->cpu = spe_cpu;
-    fr->task_blocks = 16;
-    fr->task_number = 0;
-    fr->filesize = sb.st_size;
-    fr->left_size = fr->filesize;
-    fr->division_size = 1024 * division;
-    fr->task_num = fr->filesize / (1024 * division);
-    fr->task_num += ((fr->filesize % 1024 * division) != 0);
-    fr->task_block_num = fr->task_num / fr->task_blocks;
-    fr->task_block_num += ((fr->task_num % fr->task_blocks) != 0);
-
-    HTaskPtr run = manager->create_task(RUN_READ_BLOCKS, (memaddr)&fr->self, sizeof(memaddr),0,0);
-    run->spawn();
-
-    st_mmap.file_mmap = fr->read_text;
-
-    return 0;
-}
+    int map = MAP_PRIVATE;
+    st_mmap_t st_mmap;
 
-static st_mmap_t
-my_read(TaskManager *manager,char *filename)
-{
-    /*マッピングだよ!*/
-    int fd = -1;
-    st_mmap_t st_mmap;
-    struct stat sb;
-
-    if ((fd=open(filename,O_RDONLY,0666))==0) {
-        fprintf(stderr,"can't open %s\n",filename);
-    }
-
-    if (fstat(fd,&sb)) {
-        fprintf(stderr,"can't fstat %s\n",filename);
-    }
-
-    /*sizeをページングサイズの倍数にあわせる*/
-    //st_mmap.size = fix_byte(sb.st_size,4096);
-    st_mmap.size = fix_byte(sb.st_size,division * 1024);
-    st_mmap.file_mmap = (char*)manager->allocate(st_mmap.size);
-
-    read(fd,st_mmap.file_mmap,st_mmap.size);
-
+    int fd = w->fd;
+    st_mmap.size = fix_byte(w->read_filesize,4096);
+    w->file_mmap = (char*)mmap(NULL,st_mmap.size,PROT_READ,map,fd,(off_t)0);
     if (st_mmap.file_mmap == (caddr_t)-1) {
         fprintf(stderr,"Can't mmap file\n");
+
         perror(NULL);
         exit(0);
     }
 
-    return st_mmap;
+    return 0;
 }
 
 static void
@@ -205,7 +162,7 @@
         if (use_task_array) {
             int task_num = (w->size+size-1)/size;
             if (task_num>array_task_num) task_num = array_task_num;
-            task_array = manager->create_task_array(Task_exec,task_num,1,3,1);
+            task_array = manager->create_task_array(TASK_EXEC,task_num,1,3,1);
             if (!all) {
                 t_next->wait_for(task_array);
             } else {
@@ -220,7 +177,7 @@
             if (w->size < size) size = w->size;
             if (size==0) break;
             if (use_task_array) {
-                t_exec = task_array->next_task_array(Task_exec,t_exec);
+                t_exec = task_array->next_task_array(TASK_EXEC,t_exec);
                 t_exec->set_inData(0,w->file_mmap + a*w->division_size, size);
                 t_exec->set_inData(1,w->search_word, w->search_word_len);
                 t_exec->set_inData(2,w->BMskip_table, 256);
@@ -229,7 +186,7 @@
 
                 t_exec->set_outData(0,w->o_data + a*w->out_size, w->division_out_size);
             } else if (use_compat) {
-                h_exec = manager->create_task(Task_exec);
+                h_exec = manager->create_task(TASK_EXEC);
                 h_exec->set_inData(0,w->file_mmap + i*w->division_size, size);
                 h_exec->set_outData(0,w->o_data + i*w->out_size, w->division_out_size);
 
@@ -238,7 +195,7 @@
                 h_exec->set_cpu(spe_cpu);
                 h_exec->spawn();
             } else {
-                h_exec = manager->create_task(Task_exec,
+                h_exec = manager->create_task(TASK_EXEC,
                                               (memaddr)(w->file_mmap + i*w->division_size), size,
                                               (memaddr)(w->o_data + i*w->out_size), w->division_out_size);
                 t_next->wait_for(h_exec);
@@ -310,32 +267,49 @@
     return skip_table;
 }
 
+
 static void
 run_start(TaskManager *manager, char *filename,unsigned char *search_word, int search_word_len)
 {
-    HTaskPtr t_print;
-    HTaskPtr read_run = NULL;
-    WordCount *w = (WordCount*)manager->allocate(sizeof(WordCount));
-    FilereadPtr fr = (FilereadPtr)manager->allocate(sizeof(FilereadPtr));
-
-    //fr->self = fr;
-    fr->self = fr;
-    fr->filename = filename;
+    long   fd = (long)manager->allocate(sizeof(long));
+    struct stat *sb = (struct stat*)manager->allocate(sizeof(struct stat));
+    HTaskPtr t_exec = NULL;
 
-    st_mmap_t st_mmap;
-
-    if (div_read != 0) {
-
-        read_run = manager->create_task(RUN_READ, (memaddr)&fr->self, sizeof(memaddr),0,0);
+    if ((fd=open(filename,O_RDONLY,0666))==0) {
+        fprintf(stderr,"can't open %s\n",filename);
+    }
 
-        printf("in divide read\n");
-
-    }else {
-        st_mmap = my_read(manager,filename);
-        printf("in my read\n");
+    if (fstat(fd,sb)) {
+        fprintf(stderr,"can't fstat %s\n",filename);
     }
 
-    // bzero(w,sizeof(WordCount));
+    //WordCount *w = (WordCount*)manager->allocate(sizeof(WordCount));
+    WordCountPtr w = (WordCountPtr)manager->allocate(sizeof(WordCount));
+
+    w->self = w;
+    w->fd = fd;
+    w->read_cpu = read_spe_cpu;
+    w->read_task_blocks = 16;
+    w->read_filesize = sb->st_size;
+    w->read_left_size = w->read_filesize;
+    w->read_division_size = READ_DIVISION_SIZE;
+    w->read_task_num = w->read_filesize / READ_DIVISION_SIZE;
+    w->read_task_num += ((w->read_filesize % READ_DIVISION_SIZE) != 0);
+    w->t_exec = t_exec;
+
+    printf("filesize     : %d\n",w->read_filesize);
+    printf("one_task_size: %d\n",w->read_division_size);
+    printf("task_num     : %d\n",w->read_task_num);
+
+    HTaskPtr r_run = NULL;
+
+    if (divide_read_flag != 0) {
+        char *read_text = (char*)manager->allocate(w->read_filesize);
+        r_run = manager->create_task(RUN_READ_BLOCKS, (memaddr)&w->self, sizeof(memaddr),read_text,w->read_filesize);
+        w->read_text = read_text;
+    }else {
+        r_run = manager->create_task(MMAP , (memaddr)&w->self, sizeof(memaddr),0,0);
+    }
 
     /* prepare BMSearch*/
     int *skip = (int*)manager->allocate(256 * sizeof(int));
@@ -344,13 +318,21 @@
     w->BMskip_table = create_BMskiptable(w->search_word, w->search_word_len, skip);
     w->extra_len = w->search_word_len - 1;
 
+    /* original */
+    HTaskPtr t_print;
+
+    //st_mmap_t st_mmap;
+    //st_mmap = my_mmap(filename);
+
     //w->task_blocks = blocks;
     w->self = w;
     w->task_spwaned = 0;
 
     /*sizeはdivision_sizeの倍数にしている。*/
-    w->size = w->file_size = st_mmap.size;
-    w->file_mmap = st_mmap.file_mmap;
+    w->size = w->file_size = w->read_filesize;
+    //w->file_mmap = st_mmap.file_mmap;
+    printf("w %lx\n",(long)w);
+
     /* 1task分のデータサイズ(byte) */
     if (w->size >= 1024*division) {
         w->division_size = 1024 * division;/*16kbyte*/
@@ -358,11 +340,12 @@
         w->division_size = w->size;
     }
 
-    /* exec output only "match_num" */
-    w->status_num = 1;
+    printf("dvision_size %d\n",w->division_size);
+
+    /* "word num" and "line num" */
+    w->status_num = 2;
     /* taskの数 */
     w->task_num = w->size / w->division_size;
-    printf("task num: %d\n", w->task_num);
     w->task_num = w->task_num + (w->division_size*w->task_num < w->size);
     int out_task_num = w->task_num;
 
@@ -373,33 +356,36 @@
     }
 
     w->out_task_num = out_task_num;
+    printf("task_num %d\n",w->task_num);
+    printf("out_task_num %d\n",w->out_task_num);
 
     /* out用のdivision_size. statusが2つなので、あわせて16byteになるように、long long(4byte)を使用 */
 
-    w->division_out_size = sizeof(unsigned long long)*2;
+    w->division_out_size = sizeof(unsigned long long)*4;
     int out_size = w->division_out_size*out_task_num;
     w->o_data = (unsigned long long *)manager->allocate(out_size);
-    w->out_size = 1;
+    w->out_size_ = out_size;
+    w->out_size = 4;
+    printf("out size %d\n",out_size);
 
     /*各SPEの結果を合計して出力するタスク*/
 
-    t_print = manager->create_task(Task_print,
+    t_print = manager->create_task(PRINT_TASK,
                                    (memaddr)&w->self,sizeof(memaddr),0,0);
     w->t_print = t_print;
 
-    /* Task を task_blocks ずつ起動する Task */
-    /* serialize されていると仮定する... */
-    HTaskPtr t_exec = manager->create_task(RUN_TASK_BLOCKS,
-                                           (memaddr)&w->self,sizeof(memaddr),0,0);
-
-    if (read_run != NULL) {
-        t_exec->wait_for(read_run);
-        read_run->spawn();
+    for(int i=0;i<1;i++) {
+        /* Task を task_blocks ずつ起動する Task */
+        /* serialize されていると仮定する... */
+        t_exec = manager->create_task(RUN_TASK_BLOCKS,
+                                               (memaddr)&w->self,sizeof(memaddr),0,0);
+        t_exec->wait_for(r_run);
+        t_print->wait_for(t_exec);
+        //    t_exec->iterate(4);
+        t_exec->spawn();
     }
-
-    t_print->wait_for(t_exec);
-    t_exec->iterate(2);
-
+    t_print->wait_for(r_run);
+    r_run->spawn();
     t_print->spawn();
 }
 
@@ -422,8 +408,6 @@
         } else if (strcmp(argv[i], "-c") == 0) {
             use_task_array = 0;
             use_compat = 1;
-        } else if (strcmp(argv[i], "-dr") == 0) {
-            div_read = 1;
         } else if (strcmp(argv[i], "-s") == 0) {
             use_task_array = 0;
             use_compat = 0;
@@ -440,6 +424,8 @@
             if (spe_num==0) spe_num = 1;
         } else if (strcmp(argv[i], "-sw") == 0) {
             sword = (unsigned char*)argv[i+1];
+        } else if (strcmp(argv[i], "-dr") == 0) {
+            divide_read_flag = 1;
         }
     }
     if ((filename==0) || (sword==0)) {
--- a/example/regex_mas/ppe/Exec.cc	Mon Jan 20 16:10:56 2014 +0900
+++ b/example/regex_mas/ppe/Exec.cc	Mon Jan 20 19:02:22 2014 +0900
@@ -6,7 +6,7 @@
 #define max(a,b)((a)>(b)?a:b) 
 
 /* これは必須 */
-SchedDefineTask1(Task_exec,task_exec);
+SchedDefineTask1(TASK_EXEC,task_exec);
 
 //ボイヤームーア法による文字列検索アルゴリズム
 static int BM_method(unsigned char *text,int text_len,
@@ -41,7 +41,7 @@
     int sw_len = (int)s->get_inputSize(1);
 
     //get_param
-    long task_count = (long)s->get_param(0);  //何番目のtaskか
+    //long task_count = (long)s->get_param(0);  //何番目のtaskか
     //s->printf("[exec No: %lld]\n",task_count+1);
 
     unsigned long long *o_data = (unsigned long long*)s->get_output(wbuf,0);
--- a/example/regex_mas/ppe/Print.cc	Mon Jan 20 16:10:56 2014 +0900
+++ b/example/regex_mas/ppe/Print.cc	Mon Jan 20 19:02:22 2014 +0900
@@ -5,10 +5,10 @@
 #include "WordCount.h"
 
 /* これは必須 */
-SchedDefineTask1(Task_print,task_print);
+SchedDefineTask1(PRINT_TASK,print_task);
 
 static int
-task_print(SchedTask *s, void *rbuf, void *wbuf)
+print_task(SchedTask *s, void *rbuf, void *wbuf)
 {
     WordCount *w = *(WordCount**)rbuf;
     unsigned long long *idata = w->o_data;
--- a/example/regex_mas/ppe/Read.cc	Mon Jan 20 16:10:56 2014 +0900
+++ b/example/regex_mas/ppe/Read.cc	Mon Jan 20 19:02:22 2014 +0900
@@ -11,26 +11,22 @@
 #include "Func.h"
 
 /* これは必須 */
-SchedDefineTask1(Task_read,task_read);
+SchedDefineTask1(READ_TASK,read_task);
 
 static int
-task_read(SchedTask *s, void *rbuf, void *wbuf)
+read_task(SchedTask *s, void *rbuf, void *wbuf)
 {
     long task_number = (long)s->get_param(0);   //何番目のtaskか
     long division_size = (long)s->get_param(1);
     long read_size = (long)s->get_param(2);
-    long fd = (long)s->get_param(3);    //fdの番号の受け取り
+    long fd = (long)s->get_param(3);
 
-    char *readtext = (char*)s->get_output(wbuf,0);
+    char *read_text = (char*)s->get_output(wbuf,0);
 
-    //char *text = (char*)s->allocate(sizeof(char)*read_size + 1);
-    //char text[(long)read_size];
-    //text[(long)read_size] = '\0';
-    //s->printf("[read No: %lld]\n",task_number+1);
+    pread(fd, read_text, (long)read_size , division_size*task_number);
 
-    pread(fd, readtext, read_size , division_size*task_number);
-
-    //readtext = text;
-
+    //s->printf("[start task No. %d]\n",task_number);
+    //s->printf("%s\n",read_text);
+    //s->printf("in divide_read\n");
     return 0;
 }
--- a/example/regex_mas/task_init.cc	Mon Jan 20 16:10:56 2014 +0900
+++ b/example/regex_mas/task_init.cc	Mon Jan 20 19:02:22 2014 +0900
@@ -1,34 +1,35 @@
 #include "Func.h"
 #include "Scheduler.h"
+#ifdef __CERIUM_GPU__
 #include "GpuScheduler.h"
+#endif
 
 /* 必ずこの位置に書いて */
-#ifndef __CERIUM_GPU__
-SchedExternTask(Task_exec);
-#endif
-SchedExternTask(Task_print);
-SchedExternTask(Task_read);
-SchedExternTask(RUN_READ);
+SchedExternTask(TASK_EXEC);
+SchedExternTask(READ_TASK);
 SchedExternTask(RUN_READ_BLOCKS);
+SchedExternTask(MMAP);
+SchedExternTask(TASK_EXEC);
 SchedExternTask(RUN_TASK_BLOCKS);
+SchedExternTask(PRINT_TASK);
 
 /**
  * この関数は ../spe/spe-main と違って
  * 自分で呼び出せばいい関数なので
  * 好きな関数名でおk (SchedRegisterTask は必須)
  */
+
 void
 task_init(void)
 {
 #ifdef __CERIUM_GPU__
-    GpuSchedRegister(Task_exec, "gpu/Exec.cl", "run");
-#else
-    SchedRegister(Task_exec);
+    GpuSchedRegister(TASK_EXEC, "gpu/Exec.cl", "run");
 #endif
 
-    SchedRegister(Task_read);
-    SchedRegister(Task_print);
-    SchedRegister(RUN_READ);
+    SchedRegister(TASK_EXEC);
+    SchedRegister(READ_TASK);
     SchedRegister(RUN_READ_BLOCKS);
+    SchedRegister(MMAP);
     SchedRegister(RUN_TASK_BLOCKS);
+    SchedRegister(PRINT_TASK);
 }