Mercurial > hg > Game > Cerium

comparison example/bm_search/main.cc @ 1976:a8f4227d6a21 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
rename regex_mas to bm_search
author Masataka Kohagura <e085726@ie.u-ryukyu.ac.jp>
date Mon, 03 Mar 2014 19:12:02 +0900
parents example/regex_mas/main.cc@9ebee99a9aef
children 455e620ad2b2
comparison
equal deleted inserted replaced
1975:4cf85b48ab9e 1976:a8f4227d6a21
1 #include <stdio.h>
2 #include <stdlib.h>
3 #include <string.h>
4 #include <sys/mman.h>
5 #include <sys/types.h>
6 #include <sys/stat.h>
7 #include <fcntl.h>
8 #include <unistd.h>
9 #include <sys/time.h>
10 #include <iostream>
11 #include <vector>
12 #include "TaskManager.h"
13 #include "SchedTask.h"
14 #include "Func.h"
15 #include "WordCount.h"
16
17 /* ;TODO
18 * PS3でCPU数が2以上の時に、あまりが計算されてない
19 */
20
21 extern void task_init();
22 void TMend(TaskManager *);
23 static double st_time;
24 static double ed_time;
25 int all = 0;
26 int use_task_array = 1;
27 int use_task_creater = 0;
28 int use_compat = 0;
29 int div_read = 0;
30 int array_task_num = 8;
31 int spe_num = 1;
32 int block_read_flag = 0;
33 int DIVISION_SIZE = 4*4096;
34 int READ_DIVISION_SIZE = 4*4096 * 1024;
35
36 unsigned char *sword = 0;
37 int task_count = 0;
38
39 static int division = 128; // in Kbyte
40 CPU_TYPE spe_cpu = SPE_ANY;
41 CPU_TYPE read_spe_cpu = IO_0;
42
43 const char *usr_help_str = "Usage: ./word_count [-a -c -s] [-cpu spe_num] [-sw search_word] [-file filename]\n Required filename & search_word\n";
44
45 static double
46 getTime() {
47 struct timeval tv;
48 gettimeofday(&tv, NULL);
49 return tv.tv_sec + (double)tv.tv_usec*1e-6;
50 }
51
52 typedef struct {
53 caddr_t file_mmap;
54 off_t size;
55 } st_mmap_t;
56
57 static void
58 my_mmap(char *filename, WordCount *w)
59 {
60 /*マッピングだよ!*/
61 int map = MAP_PRIVATE;
62 st_mmap_t st_mmap;
63 struct stat sb;
64 long fd = w->fd;
65
66 if ((fd=open(filename,O_RDONLY,0666))==0) {
67 fprintf(stderr,"can't open %s\n",filename);
68 }
69
70 if (fstat(fd,&sb)) {
71 fprintf(stderr,"can't fstat %s\n",filename);
72 }
73
74 //w->file_mmap = (char*)mmap(NULL,w->read_filesize,PROT_READ,map,fd,(off_t)0);
75 madvise(w->file_mmap, w->read_filesize, POSIX_MADV_NORMAL);
76 w->file_mmap = (char*)mmap(NULL,w->read_filesize,PROT_READ,map,fd,(off_t)0);
77
78 if (st_mmap.file_mmap == (caddr_t)-1) {
79 fprintf(stderr,"Can't mmap file\n");
80 perror(NULL);
81 exit(0);
82 }
83
84 return ;
85 }
86
87
88 static void
89 run_tasks(SchedTask *manager, WordCount *w, int task_count,HTaskPtr t_read, HTaskPtr t_next, int size)
90 {
91
92 if (task_count < array_task_num) {
93 array_task_num = task_count;
94 if (task_count<=0) return;
95 }
96 for (int i = 0; i < task_count; i += array_task_num) {
97 HTask *task_array;
98 if (use_task_array) {
99 int task_num = (w->size+size-1)/size;
100 if (task_num>array_task_num) task_num = array_task_num;
101 task_array = manager->create_task_array(TASK_EXEC,task_num,1,3,1);
102 if (t_read != 0) task_array->wait_for(t_read);
103 if (!all) {
104 t_next->wait_for(task_array);
105 } else {
106 w->t_print->wait_for(task_array);
107 }
108 }
109
110 Task *t_exec = 0;
111 HTask *h_exec = 0;
112 for (int j = 0; j < array_task_num; j++) {
113 long i = w->task_spawned++;
114 if (w->size < size) size = w->size;
115 if (size==0) break;
116 if (use_task_array) {
117 t_exec = task_array->next_task_array(TASK_EXEC,t_exec);
118 t_exec->set_inData(0,w->file_mmap + i*w->division_size, size);
119 t_exec->set_inData(1,w->search_word, w->search_word_len);
120 t_exec->set_inData(2,w->BMskip_table, 256);
121
122 t_exec->set_param(0,(long)i);
123
124 t_exec->set_outData(0,w->o_data + i*w->out_size, w->division_out_size);
125 } else if (use_compat) {
126 h_exec = manager->create_task(TASK_EXEC);
127 if (t_read != 0) h_exec->wait_for(t_read);
128 h_exec->set_inData(0,w->file_mmap + i*w->division_size, size);
129 h_exec->set_outData(0,w->o_data + i*w->out_size, w->division_out_size);
130
131 t_next->wait_for(h_exec);
132
133 h_exec->set_cpu(spe_cpu);
134 h_exec->spawn();
135 } else {
136 h_exec = manager->create_task(TASK_EXEC,
137 (memaddr)(w->file_mmap + i*w->division_size), size,
138 (memaddr)(w->o_data + i*w->out_size), w->division_out_size);
139 if (t_read != 0) h_exec->wait_for(t_read);
140 t_next->wait_for(h_exec);
141 h_exec->set_cpu(spe_cpu);
142 h_exec->spawn();
143 }
144 w->size -= w->division_size;
145 w->task_num--;
146 }
147 if (use_task_array) {
148 task_array->spawn_task_array(t_exec->next());
149 task_array->set_cpu(spe_cpu);
150 task_array->spawn();
151 } else {
152 //if (!all) t_next->wait_for(h_exec);
153 }
154 }
155 }
156
157 /**
158 * このTaskは、PPE上で実行されるので、並列に実行されることはない
159 * 二つ実行されていて、Task が足りなくなることがないようにしている。
160 */
161
162 SchedDefineTask1(BREAD_RUN_TASK_BLOCKS,bread_run16);
163
164 static int
165 bread_run16(SchedTask *manager, void *in, void *out)
166 {
167 WordCount *w = *(WordCount **)in;
168
169 HTaskPtr t_read = manager->create_task(READ_TASK);
170 w->t_print->wait_for(t_read);
171 t_read->set_cpu(read_spe_cpu);
172 t_read->set_param(0,w->fd);
173
174 if (w->task_num < w->task_blocks) {
175 t_read->set_param(1,w->task_spawned*w->division_size);
176 t_read->set_outData(0, w->file_mmap + w->task_spawned * w->division_size, w->task_blocks * w->division_size);
177
178 // last case
179 while (w->size >= w->division_size)
180 run_tasks(manager,w,w->task_num,t_read,w->t_print, w->division_size + w->extra_len);
181 // remaining data
182 while (w->size>0)
183 run_tasks(manager,w,1,t_read,w->t_print, w->size);
184
185 t_read->set_param(2,w->task_spawned*w->division_size);
186 t_read->spawn();
187 } else {
188 HTaskPtr t_next = manager->create_task(BREAD_RUN_TASK_BLOCKS,
189 (memaddr)&w->self,sizeof(memaddr),0,0);
190 w->t_print->wait_for(t_next);
191
192 t_read->set_param(1,w->task_spawned*w->division_size);
193 t_read->set_outData(0, w->file_mmap + w->task_spawned * w->division_size, w->task_blocks * w->division_size);
194
195 run_tasks(manager,w, w->task_blocks, t_read, t_next, w->division_size + w->extra_len);
196
197 t_read->set_param(2,w->task_spawned*w->division_size + w->extra_len);
198
199 t_read->spawn();
200 t_next->spawn();
201 }
202 return 0;
203 }
204
205 SchedDefineTask1(MMAP_RUN_TASK_BLOCKS,mmap_run16);
206
207 static int
208 mmap_run16(SchedTask *manager, void *in, void *out)
209 {
210 WordCount *w = *(WordCount **)in;
211
212 if (w->task_num < w->task_blocks) {
213 // last case
214 while (w->size >= w->division_size)
215 run_tasks(manager,w,w->task_num,0, w->t_print, w->division_size + w->extra_len);
216 // remaining data
217 while (w->size>0)
218 run_tasks(manager,w,1,0,w->t_print, w->size);
219 } else {
220 HTaskPtr t_next = manager->create_task(MMAP_RUN_TASK_BLOCKS,
221 (memaddr)&w->self,sizeof(memaddr),0,0);
222 w->t_print->wait_for(t_next);
223
224 run_tasks(manager,w, w->task_blocks,0, t_next, w->division_size + w->extra_len);
225
226 t_next->spawn();
227 }
228 return 0;
229 }
230
231
232 static int blocks = 48;
233 //static int blocks = 31 * 6 * 24;
234
235 //Boyer Moore法に使用するテーブルを作成
236 static int*
237 create_BMskiptable(unsigned char *search_word,int search_word_len,int *skip_table)
238 {
239 for(int i = 0; i < 256; ++i){
240 skip_table[i] = search_word_len;
241 }
242
243 for(int j = 0; j < search_word_len - 1; ++j){
244 skip_table[search_word[j]] = search_word_len - j - 1;
245 }
246 return skip_table;
247 }
248
249
250 static void
251 run_start(TaskManager *manager, char *filename,unsigned char *search_word, int search_word_len)
252 {
253 long fd = (long)manager->allocate(sizeof(long));
254 struct stat *sb = (struct stat*)manager->allocate(sizeof(struct stat));
255 HTaskPtr t_exec;
256
257 if ((fd=open(filename,O_RDONLY,0666))==0) {
258 fprintf(stderr,"can't open %s\n",filename);
259 return ;
260 }
261
262 if (fstat(fd,sb)) {
263 fprintf(stderr,"can't fstat %s\n",filename);
264 return ;
265 }
266
267 WordCountPtr w = (WordCountPtr)manager->allocate(sizeof(WordCount));
268
269 w->self = w;
270 w->fd = fd;
271 w->read_filesize = sb->st_size;
272
273
274 if (block_read_flag == 1) {
275 printf("[block read mode]\n");
276 w->file_mmap = (char*)manager->allocate(w->read_filesize);
277 }else {
278 printf("[mmap mode]\n");
279 my_mmap(filename, w);
280 }
281
282
283 /* prepare BMSearch*/
284 int *skip = (int*)manager->allocate(256 * sizeof(int)); // 文字列に対応した table を用意
285 w->search_word = search_word;
286 w->search_word_len = search_word_len;
287 w->BMskip_table = create_BMskiptable(w->search_word, w->search_word_len, skip);
288 w->extra_len = w->search_word_len - 1;
289
290 /* original */
291 HTaskPtr t_print;
292
293 //w->task_blocks = blocks;
294 w->self = w;
295 w->task_spawned = 0;
296
297 w->size = w->file_size = w->read_filesize;
298 printf("w %lx\n",(long)w);
299
300 /* 1task分のデータサイズ(byte) */
301 if (w->size >= 1024*division) {
302 w->division_size = 1024 * division;/*16kbyte*/
303 } else {
304 w->division_size = w->size;
305 }
306
307 printf("division_size %ld\n",w->division_size);
308
309 /* "word num" and "line num" */
310 w->status_num = 2;
311 /* taskの数 */
312 w->task_num = w->size / w->division_size;
313 w->task_num = w->task_num + (w->division_size*w->task_num < w->size);
314 int out_task_num = w->task_num;
315
316 if(!all) {
317 w->task_blocks = blocks;
318 } else {
319 w->task_blocks = w->task_num;
320 }
321
322 w->out_task_num = out_task_num;
323 printf("task_num %ld\n",w->task_num);
324 printf("out_task_num %ld\n",w->out_task_num);
325
326 /* out用のdivision_size. statusが2つなので、あわせて16byteになるように、long long(4byte)を使用 */
327
328 w->division_out_size = sizeof(unsigned long long)*1;
329 int out_size = w->division_out_size*out_task_num;
330 w->o_data = (unsigned long long *)manager->allocate(out_size);
331 w->out_size_ = out_size;
332 w->out_size = 1;
333 printf("out size %d\n",out_size);
334
335 /*各SPEの結果を合計して出力するタスク*/
336
337 t_print = manager->create_task(PRINT_TASK,
338 (memaddr)&w->self,sizeof(memaddr),0,0);
339 w->t_print = t_print;
340
341 for(int i=0;i<1;i++) {
342 /* Task を task_blocks ずつ起動する Task */
343 /* serialize されていると仮定する... */
344 if (block_read_flag == 1) {
345 t_exec = manager->create_task(BREAD_RUN_TASK_BLOCKS,
346 (memaddr)&w->self,sizeof(memaddr),0,0);
347 }else {
348 t_exec = manager->create_task(MMAP_RUN_TASK_BLOCKS,
349 (memaddr)&w->self,sizeof(memaddr),0,0);
350 }
351
352 t_print->wait_for(t_exec);
353 // t_exec->iterate(4);
354 t_exec->spawn();
355 }
356 w->t_exec = t_exec;
357
358 t_print->spawn();
359 }
360
361 static char*
362 init(int argc, char **argv)
363 {
364
365 char *filename = 0;
366
367 for (int i = 1; argv[i]; ++i) {
368 if (strcmp(argv[i], "-file") == 0) {
369 filename = argv[i+1];
370 } else if (strcmp(argv[i], "-division") == 0) {
371 division = atoi(argv[i+1]);
372 } else if (strcmp(argv[i], "-block") == 0) {
373 blocks = atoi(argv[i+1]);
374 } else if (strcmp(argv[i], "-a") == 0) {
375 // create task all at once
376 all = 1;
377 } else if (strcmp(argv[i], "-c") == 0) {
378 use_task_array = 0;
379 use_compat = 1;
380 } else if (strcmp(argv[i], "-s") == 0) {
381 use_task_array = 0;
382 use_compat = 0;
383 } else if (strcmp(argv[i], "-t") == 0) {
384 use_task_creater = 1;
385 use_task_array = 0;
386 use_compat = 0;
387 } else if (strcmp(argv[i], "-anum") == 0) {
388 array_task_num = atoi(argv[i+1]);
389 } else if (strcmp(argv[i], "-g") == 0 ) {
390 spe_cpu = GPU_0;
391 } else if (strcmp(argv[i], "-cpu") == 0) {
392 spe_num = atoi(argv[i+1]);
393 if (spe_num==0) spe_num = 1;
394 } else if (strcmp(argv[i], "-sw") == 0) {
395 sword = (unsigned char*)argv[i+1];
396 } else if (strcmp(argv[i], "-br") == 0) {
397 block_read_flag = 1;
398 }
399 }
400 if ((filename==0) || (sword==0)) {
401 puts(usr_help_str);
402 exit(1);
403 }
404
405 return filename;
406 }
407
408
409 int
410 TMmain(TaskManager *manager, int argc, char *argv[])
411 {
412
413 char *filename = init(argc, argv);
414 int sw_len = strlen((const char *)sword);
415
416 unsigned char *search_word = (unsigned char*)manager->allocate(sw_len + 1);
417 memcpy(search_word, sword, sw_len + 1);
418
419 task_init();
420 st_time = getTime();
421 run_start(manager, filename, search_word, sw_len);
422 manager->set_TMend(TMend);
423 return 0;
424 }
425
426 void
427 TMend(TaskManager *manager)
428 {
429 ed_time = getTime();
430 printf("Time: %0.6f\n",ed_time-st_time);
431 }
432
433 /* end */