# HG changeset patch # User anatofuz # Date 1585765141 -32400 # Node ID cc568d791a2832818d60ed21f2579dfe38031a89 # Parent 8313f8fd6a44e80ac114429a0ff97ee7efbdf9ea fix diff -r 8313f8fd6a44 -r cc568d791a28 2020/previous/dumpFromPDFWHTML.json --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/2020/previous/dumpFromPDFWHTML.json Thu Apr 02 03:19:01 2020 +0900 @@ -0,0 +1,1 @@ +[{"ID":"610002071","Name":"工業数学Ⅰ 07組","Day":[3,4],"DayOfWeek":0,"Teacher":"山田 孝治","Grades":[1],"Place":"工1-321"},{"ID":"610002071","Name":"工業数学Ⅰ 07組","Day":[3,4],"DayOfWeek":0,"Teacher":"山田 孝治","Grades":[1],"Place":"工1-321"},{"ID":"610000071","Name":"工学基礎演習 06組","Day":[3,4],"DayOfWeek":0,"Teacher":"國田　樹　[Itsuki Kunita], 遠藤 聡志","Grades":[1],"Place":"工1-321"},{"ID":"610000071","Name":"工学基礎演習 06組","Day":[3,4],"DayOfWeek":0,"Teacher":"國田　樹　[Itsuki Kunita], 遠藤 聡志","Grades":[1],"Place":"工1-321"},{"ID":"617009001","Name":"プロジェクトデザイン","Day":[2],"DayOfWeek":3,"Teacher":"山田 孝治","Grades":[2],"Place":"工1-321"},{"ID":"610000071","Name":"工学基礎演習 06組","Day":[3,4],"DayOfWeek":0,"Teacher":"國田　樹　[Itsuki Kunita], 遠藤 聡志","Grades":[1],"Place":"工1-321"},{"ID":"610000071","Name":"工学基礎演習 06組","Day":[3,4],"DayOfWeek":0,"Teacher":"國田　樹　[Itsuki Kunita], 遠藤 聡志","Grades":[1],"Place":"工1-321"},{"ID":"610019001","Name":"産業社会学原論Ⅰ","Day":[5],"DayOfWeek":4,"Teacher":"玉城 史朗","Grades":[3],"Place":"工1-322"},{"ID":"610019001","Name":"産業社会学原論Ⅰ","Day":[5],"DayOfWeek":4,"Teacher":"玉城 史朗","Grades":[3],"Place":"工1-322"},{"ID":"617000001","Name":"プログラミング演習Ⅰ","Day":[2],"DayOfWeek":4,"Teacher":"赤嶺 有平　[Akamine Yuhei]","Grades":[1],"Place":"工1-321"},{"ID":"610004071","Name":"プログラミングⅠ","Day":[1],"DayOfWeek":4,"Teacher":"當間 愛晃","Grades":[1],"Place":"工1-321"},{"ID":"610004071","Name":"プログラミングⅠ","Day":[1],"DayOfWeek":4,"Teacher":"當間 愛晃","Grades":[1],"Place":"工1-321"},{"ID":"617003011","Name":"知能情報実験Ⅰ 01組","Day":[3,4],"DayOfWeek":1,"Teacher":"吉田 たけお, 宮里 智樹, 赤嶺 有平　[Akamine Yuhei], 國田　樹　[Itsuki Kunita]","Grades":[2],"Place":"地創棟508"},{"ID":"610005071","Name":"基礎数学Ⅰ 06組","Day":[3],"DayOfWeek":3,"Teacher":"岡﨑 威生, 國田　樹　[Itsuki Kunita]","Grades":[1],"Place":"工1-321"},{"ID":"610005071","Name":"基礎数学Ⅰ 06組","Day":[3],"DayOfWeek":3,"Teacher":"岡﨑 威生, 國田　樹　[Itsuki Kunita]","Grades":[1],"Place":"工1-321"},{"ID":"617015001","Name":"ディジタル回路","Day":[4],"DayOfWeek":0,"Teacher":"吉田 たけお","Grades":[2],"Place":"工1-322"},{"ID":"617005001","Name":"アルゴリズムとデータ構造","Day":[2],"DayOfWeek":2,"Teacher":"名嘉村 盛和","Grades":[2],"Place":"工1-322"},{"ID":"610012071","Name":"技術者の倫理 03組","Day":[1],"DayOfWeek":3,"Teacher":"和田 知久","Grades":[1],"Place":"工1-221"},{"ID":"610012071","Name":"技術者の倫理 03組","Day":[1],"DayOfWeek":3,"Teacher":"和田 知久","Grades":[1],"Place":"工1-221"},{"ID":"617003021","Name":"知能情報実験Ⅰ 02組","Day":[3,4],"DayOfWeek":4,"Teacher":"吉田 たけお, 宮里 智樹, 赤嶺 有平　[Akamine Yuhei], 國田　樹　[Itsuki Kunita]","Grades":[2],"Place":"地創棟508"},{"ID":"101501081","Name":"大学英語 08組","Day":[2],"DayOfWeek":0,"Teacher":"富里　明美","Grades":[1],"Place":"共3-205"},{"ID":"101501081","Name":"大学英語 08組","Day":[2],"DayOfWeek":3,"Teacher":"富里　明美","Grades":[1],"Place":"共3-403"},{"ID":"617006001","Name":"情報ネットワークⅠ","Day":[1],"DayOfWeek":2,"Teacher":"長田 智和","Grades":[2],"Place":"工1-322"},{"ID":"610013071","Name":"キャリアデザイン 05組","Day":[4,5],"DayOfWeek":1,"Teacher":"遠藤 聡志","Grades":[3],"Place":"工1-322"},{"ID":"610013071","Name":"キャリアデザイン 05組","Day":[4,5],"DayOfWeek":1,"Teacher":"遠藤 聡志","Grades":[3],"Place":"工1-322"},{"ID":"610001011","Name":"キャリアデザイン入門 01組","Day":[5],"DayOfWeek":1,"Teacher":"清水 肇, 渡嘉敷 健","Grades":[1],"Place":"工1-221"},{"ID":"610042001","Name":"情報科教育法B","Day":[1],"DayOfWeek":3,"Teacher":"岡﨑 威生","Grades":[4],"Place":"工1-509"},{"ID":"610042001","Name":"情報科教育法B","Day":[1],"DayOfWeek":3,"Teacher":"岡﨑 威生","Grades":[4],"Place":"工1-509"},{"ID":"100408031","Name":"物理学Ⅰ 03組","Day":[2],"DayOfWeek":2,"Teacher":"深水 孝則","Grades":[4],"Place":"共1-118"},{"ID":"617021001","Name":"知能情報実験III","Day":[3,4],"DayOfWeek":3,"Teacher":"國田　樹　[Itsuki Kunita], 全教員","Grades":[3],"Place":"地創棟508"},{"ID":"610001021","Name":"キャリアデザイン入門 02組","Day":[5],"DayOfWeek":1,"Teacher":"清水 肇, 渡嘉敷 健","Grades":[1],"Place":"工1-222"},{"ID":"617028001","Name":"ヒューマンコンピュータインタラクション","Day":[2],"DayOfWeek":1,"Teacher":"赤嶺 有平　[Akamine Yuhei]","Grades":[3],"Place":"工2-313"},{"ID":"101501081","Name":"大学英語 08組","Day":[2],"DayOfWeek":0,"Teacher":"富里　明美","Grades":[1],"Place":"共3-205"},{"ID":"101501081","Name":"大学英語 08組","Day":[2],"DayOfWeek":3,"Teacher":"富里　明美","Grades":[1],"Place":"共3-403"},{"ID":"617038001","Name":"ソフトウエア開発演習III","Day":[3],"DayOfWeek":2,"Teacher":"河野 真治","Grades":[3],"Place":"工1-321"},{"ID":"617022001","Name":"ソフトウェア工学","Day":[4],"DayOfWeek":2,"Teacher":"河野 真治","Grades":[3],"Place":"工1-321"},{"ID":"101501091","Name":"大学英語 09組","Day":[2],"DayOfWeek":0,"Teacher":"宮城　和文","Grades":[1],"Place":"共3-403"},{"ID":"101501091","Name":"大学英語 09組","Day":[2],"DayOfWeek":3,"Teacher":"宮城　和文","Grades":[1],"Place":"共3-201"},{"ID":"610001041","Name":"キャリアデザイン入門 04組","Day":[5],"DayOfWeek":1,"Teacher":"清水 肇, 渡嘉敷 健","Grades":[1],"Place":"工1-222"},{"ID":"610001031","Name":"キャリアデザイン入門 03組","Day":[5],"DayOfWeek":1,"Teacher":"清水 肇, 渡嘉敷 健","Grades":[1],"Place":"工1-221"},{"ID":"617008001","Name":"データサイエンス基礎","Day":[1,2],"DayOfWeek":1,"Teacher":"岡﨑 威生","Grades":[2],"Place":"工1-321"},{"ID":"617039001","Name":"ICT実践英語I","Day":[1],"DayOfWeek":2,"Teacher":"比嘉　健伸","Grades":[4],"Place":"共4-305"},{"ID":"617023001","Name":"数理計画とアルゴリズム","Day":[2],"DayOfWeek":2,"Teacher":"長山 格","Grades":[3],"Place":"工1-321"},{"ID":"R00409001","Name":"システムアーキテクチャ論","Day":[1],"DayOfWeek":2,"Teacher":"和田 知久","Grades":[5],"Place":"地創棟503"},{"ID":"101501091","Name":"大学英語 09組","Day":[2],"DayOfWeek":0,"Teacher":"宮城　和文","Grades":[1],"Place":"共3-403"},{"ID":"101501091","Name":"大学英語 09組","Day":[2],"DayOfWeek":3,"Teacher":"宮城　和文","Grades":[1],"Place":"共3-201"},{"ID":"610013071","Name":"キャリアデザイン 05組","Day":[4,5],"DayOfWeek":1,"Teacher":"遠藤 聡志","Grades":[3],"Place":"工1-322"},{"ID":"610013071","Name":"キャリアデザイン 05組","Day":[4,5],"DayOfWeek":1,"Teacher":"遠藤 聡志","Grades":[3],"Place":"工1-322"},{"ID":"R00079001","Name":"ソフトウェアシステム論","Day":[4],"DayOfWeek":3,"Teacher":"河野 真治","Grades":[5],"Place":"工1-321"},{"ID":"610016001","Name":"技術英語Ⅰ","Day":[1],"DayOfWeek":4,"Teacher":"比嘉 広樹","Grades":[3],"Place":"工4-111"},{"ID":"610002071","Name":"工業数学Ⅰ 07組","Day":[3,4],"DayOfWeek":0,"Teacher":"山田 孝治","Grades":[1],"Place":"工1-321"},{"ID":"610002071","Name":"工業数学Ⅰ 07組","Day":[3,4],"DayOfWeek":0,"Teacher":"山田 孝治","Grades":[1],"Place":"工1-321"},{"ID":"R01768001","Name":"UI/UX論","Day":[5],"DayOfWeek":1,"Teacher":"赤嶺 有平　[Akamine Yuhei]","Grades":[5],"Place":"工1-321"},{"ID":"610019001","Name":"産業社会学原論Ⅰ","Day":[5],"DayOfWeek":4,"Teacher":"玉城 史朗","Grades":[3],"Place":"工1-322"},{"ID":"610019001","Name":"産業社会学原論Ⅰ","Day":[5],"DayOfWeek":4,"Teacher":"玉城 史朗","Grades":[3],"Place":"工1-322"},{"ID":"610004071","Name":"プログラミングⅠ","Day":[1],"DayOfWeek":4,"Teacher":"當間 愛晃","Grades":[1],"Place":"工1-321"},{"ID":"610004071","Name":"プログラミングⅠ","Day":[1],"DayOfWeek":4,"Teacher":"當間 愛晃","Grades":[1],"Place":"工1-321"},{"ID":"100826081","Name":"微分積分学ＳＴ（スタンダードコース）Ⅰ 08組","Day":[3],"DayOfWeek":2,"Teacher":"林　正史","Grades":[1],"Place":"共1-217"},{"ID":"617007001","Name":"コンピュータシステム","Day":[2],"DayOfWeek":0,"Teacher":"姜 東植","Grades":[2],"Place":"工1-321"},{"ID":"R01423001","Name":"プロジェクト・マネジメント演習","Day":[5],"DayOfWeek":4,"Teacher":"名嘉村 盛和","Grades":[5],"Place":"地創棟508"},{"ID":"R00087001","Name":"知能ロボット論","Day":[5],"DayOfWeek":3,"Teacher":"山田 孝治","Grades":[5],"Place":"地創棟508"},{"ID":"610042001","Name":"情報科教育法B","Day":[1],"DayOfWeek":3,"Teacher":"岡﨑 威生","Grades":[4],"Place":"工1-509"},{"ID":"610042001","Name":"情報科教育法B","Day":[1],"DayOfWeek":3,"Teacher":"岡﨑 威生","Grades":[4],"Place":"工1-509"},{"ID":"610005071","Name":"基礎数学Ⅰ 06組","Day":[3],"DayOfWeek":3,"Teacher":"岡﨑 威生, 國田　樹　[Itsuki Kunita]","Grades":[1],"Place":"工1-321"},{"ID":"610005071","Name":"基礎数学Ⅰ 06組","Day":[3],"DayOfWeek":3,"Teacher":"岡﨑 威生, 國田　樹　[Itsuki Kunita]","Grades":[1],"Place":"工1-321"},{"ID":"610038001","Name":"技術英語Ⅲ","Day":[1],"DayOfWeek":3,"Teacher":"小野 尋子","Grades":[4],"Place":"共4-505"},{"ID":"R01672001","Name":"データマイニング論","Day":[4],"DayOfWeek":1,"Teacher":"當間 愛晃","Grades":[5],"Place":"工1-321"},{"ID":"617016001","Name":"情報処理技術概論","Day":[3],"DayOfWeek":0,"Teacher":"宮里 智樹","Grades":[2],"Place":"工1-322"},{"ID":"610002021","Name":"工業数学Ⅰ 02組","Day":[2],"DayOfWeek":0,"Teacher":"宮﨑 達二郎","Grades":[1],"Place":"工4-111"},{"ID":"610012071","Name":"技術者の倫理 03組","Day":[1],"DayOfWeek":3,"Teacher":"和田 知久","Grades":[1],"Place":"工1-221"},{"ID":"610012071","Name":"技術者の倫理 03組","Day":[1],"DayOfWeek":3,"Teacher":"和田 知久","Grades":[1],"Place":"工1-221"},{"ID":"R00084001","Name":"知能システム論","Day":[2],"DayOfWeek":2,"Teacher":"姜 東植","Grades":[5],"Place":"地創棟504"}] \ No newline at end of file diff -r 8313f8fd6a44 -r cc568d791a28 cmd_donwload.go --- a/cmd_donwload.go Wed Apr 01 19:18:40 2020 +0900 +++ b/cmd_donwload.go Thu Apr 02 03:19:01 2020 +0900 @@ -1,9 +1,15 @@ package lectable import ( + "bufio" "context" + "encoding/json" "fmt" "io" + "os" + "path/filepath" + "strconv" + "strings" "www.cr.ie.u-ryukyu.ac.jp/hg/Members/anatofuz/lectable/syllabus" ) @@ -18,6 +24,34 @@ return "donwload html from lecture ids" } +func convertPDFJson(jsonPath string) ([]DumpLectureFPDF, error) { + f, err := os.Open(jsonPath) + if err != nil { + return nil, err + } + br := bufio.NewReader(f) + dec := json.NewDecoder(br) + var dlf []DumpLectureFPDF + err = dec.Decode(&dlf) + if err != nil { + return nil, err + } + return dlf, nil +} + +func strToIntDays(s string) (int, int, error) { + for dayIndex, dow := range days { + if strings.Contains(s, dow) { + d, err := strconv.Atoi(s[kanjiDayLen:]) + if err != nil { + return 0, 0, err + } + return dayIndex, d, nil + } + } + return 0, 0, nil +} + func (cd *cmdDownload) run(ctx context.Context, argv []string, outStream, errStream io.Writer) error { fmt.Println("download now!!") dh := syllabus.CreateGetSyllabus() @@ -25,12 +59,83 @@ if err != nil { return err } - arr3 := []string{"601495001", "600625001"} + + dlf, err := convertPDFJson(argv[0]) + if err != nil { + return err + } + + var lecIDS []string + //var idToLecPDFJ map[string][]Lec + idToLecPDFJ := make(map[string][]Lec, len(dlf)) - lwps, err := dh.LecIDStoDonwlodSyllabus(ctx, arr3, outStream) + for _, dl := range dlf { + lecIDS = append(lecIDS, dl.Id) + //idToLecPDFJ[dl.Id] = append(idToLecPDFJ[dl.Id], Lec{ID: dl.Id, Grades: dl.Grades, isSelect: dl.IsSelect, Place: dl.Place}) + le := Lec{ID: dl.Id, Grades: dl.Grades, isSelect: dl.IsSelect, Place: dl.Place} + if dl.Day != "" { + dow, dindex, err := strToIntDays(dl.Day) + if err != nil { + return err + } + le.DayOfWeek = dow + le.Day = []int{dindex} + } + idToLecPDFJ[dl.Id] = append(idToLecPDFJ[dl.Id], le) + } + + lwps, err := dh.LecIDStoDonwlodSyllabus(ctx, lecIDS, outStream) lectures, err := dh.LectureWPathS2LectureStruct(ctx, lwps, outStream) if err != nil { return err } - return dh.DumpLecureToJson(lectures) + + var lecs []Lec + + for _, lect := range *lectures { + fpds := idToLecPDFJ[lect.ID] + for _, fp := range fpds { + for _, day := range lect.Day { + fp.Name = lect.Name + if fp.Day == nil { + fp.Day = []int{day.Period} + if day.LastPeriod != -1 { + fp.Day = append(fp.Day, day.LastPeriod) + } + fp.DayOfWeek = day.DayOfWeek + } + fp.Teacher = lect.Teacher + lecs = append(lecs, fp) + } + } + } + return dumpLecToJson(lecs, dh.Outputdir) } + +func dumpLecToJson(lec []Lec, outputdir string) error { + bytes, err := json.Marshal(lec) + if err != nil { + return err + } + fp := filepath.Join(outputdir, "dumpFromPDFWHTML.json") + file, err := os.Create(fp) + bw := bufio.NewWriter(file) + _, err = bw.Write(bytes) + if err != nil { + return err + } + bw.Flush() + file.Close() + return nil +} + +type Lec struct { + ID string `json:id` + Name string `json:name` + Day []int `json:day` + DayOfWeek int `json:dayOfWeek` + Teacher string `json:teacher` + isSelect bool `json:isSelect` + Grades []int `json:grades` + Place string `json:place` +} diff -r 8313f8fd6a44 -r cc568d791a28 syllabus/getSyllabus.go --- a/syllabus/getSyllabus.go Wed Apr 01 19:18:40 2020 +0900 +++ b/syllabus/getSyllabus.go Thu Apr 02 03:19:01 2020 +0900 @@ -42,10 +42,10 @@ //Lecture ID is ex. 600625001 , Name is プログラミング1, Day is LecutreDay type Lecture struct { - ID string `json: "id"` - Name string `json: "name` - Day LectureDay `json: "day"` - Teacher string `json: "teacher"` + ID string `json: "id"` + Name string `json: "name` + Day []LectureDay `json: "day"` + Teacher string `json: "teacher"` } type LectureWPath struct { @@ -57,14 +57,12 @@ func CreateGetSyllabus() *GetSyllabus { var gs GetSyllabus tm := time.Now() - //gs.year = tm.Year() - gs.year = 2019 + gs.year = tm.Year() if tm.Month() < 7 { gs.term = "previous" } else { gs.term = "latter" } - gs.term = "previous" gs.Outputdir = filepath.Join(strconv.Itoa(gs.year), gs.term) return &gs @@ -153,22 +151,33 @@ } func (g *GetSyllabus) LectureWPathS2LectureStruct(ctx context.Context, lwps *[]LectureWPath, outStream io.Writer) (*[]Lecture, error) { - //var wg sync.WaitGroup - ch := make(chan Lecture, len(*lwps)) - for _, lwp := range *lwps { - //wg.Add(1) - go func(lwp LectureWPath) { - //defer wg.Done() - lec, _ := g.LectureWPath2LectureStruct(&lwp) - ch <- *lec - }(lwp) - } - //wg.Wait() + /* + ch := make(chan Lecture, len(*lwps)) + for _, lwp := range *lwps { + //wg.Add(1) + go func(lwp LectureWPath) { + //defer wg.Done() + lec, _ := g.LectureWPath2LectureStruct(&lwp) + ch <- *lec + }(lwp) + } + //wg.Wait() + + var lecs []Lecture + for range *lwps { + lecs = append(lecs, <-ch) + } + */ var lecs []Lecture - for range *lwps { - lecs = append(lecs, <-ch) + for _, lwp := range *lwps { + lec, err := g.LectureWPath2LectureStruct(&lwp) + if err != nil { + return nil, err + } + lecs = append(lecs, *lec) } + return &lecs, nil } @@ -193,22 +202,36 @@ i += len(dayPeriodID) day := line[i:j] - lec.Day.DayOfWeek = kanjiday2int(day[0:dayOfWeeklen]) + var lecDay LectureDay + + lecDay.DayOfWeek = kanjiday2int(day[0:dayOfWeeklen]) if err != nil { - return nil, errors.Wrap(err, "failed convert day") + fmt.Printf("%s is noconvert, day: %s\n", lwp.Path, day) + return nil, errors.Wrap(err, "failed convert dayOfWeek") } - lec.Day.Period, err = strconv.Atoi(day[dayOfWeeklen : dayOfWeeklen+1]) + lecDay.Period, err = strconv.Atoi(day[dayOfWeeklen : dayOfWeeklen+1]) if err != nil { - return nil, errors.Wrap(err, "failed convert day") + fmt.Printf("%s is noconvert, day: %s\n", lwp.Path, day) + return nil, errors.Wrap(err, "failed convert period") } if len(day) != (dayOfWeeklen + 1) { // dayOfWeeklen + 1 == 月3, 火2 - lec.Day.LastPeriod, err = strconv.Atoi(day[dayOfWeeklen+4:]) // 4 is \d + 〜 + //月2,木3 + if nextKanji := strings.Index(day, ","); nextKanji != -1 { + lec.Day = append(lec.Day, lecDay) + continue + } + + // 月3~4 + lecDay.LastPeriod, err = strconv.Atoi(day[dayOfWeeklen+4:]) // 4 is \d + 〜 + lec.Day = append(lec.Day, lecDay) if err != nil { + fmt.Printf("%s is noconvert, day: %s\n", lwp.Path, day) return nil, errors.Wrap(err, "failed convert day") } continue } - lec.Day.LastPeriod = -1 + lecDay.LastPeriod = -1 + lec.Day = append(lec.Day, lecDay) continue } }