Mercurial > hg > Members > anatofuz > lectable
changeset 6:fca852bfd500
...
author | anatofuz <anatofuz@cr.ie.u-ryukyu.ac.jp> |
---|---|
date | Tue, 31 Mar 2020 15:12:16 +0900 |
parents | a0d23f38344d |
children | 514dc6c6a683 |
files | cmd_donwload.go syllabus/getSyllabus.go |
diffstat | 2 files changed, 77 insertions(+), 42 deletions(-) [+] |
line wrap: on
line diff
--- a/cmd_donwload.go Tue Mar 31 13:40:06 2020 +0900 +++ b/cmd_donwload.go Tue Mar 31 15:12:16 2020 +0900 @@ -27,6 +27,14 @@ } arr3 := []string{"601495001", "600625001"} - err = dh.LecIDStoDonwlodSyllabus(ctx, arr3) + lwps, err := dh.LecIDStoDonwlodSyllabus(ctx, arr3) + fmt.Println(lwps) + for _, lwp := range *lwps { + lec, err := dh.LecIDwFilePath2LectureStruct(&lwp) + if err != nil { + return err + } + fmt.Println(lec) + } return err }
--- a/syllabus/getSyllabus.go Tue Mar 31 13:40:06 2020 +0900 +++ b/syllabus/getSyllabus.go Tue Mar 31 15:12:16 2020 +0900 @@ -3,6 +3,7 @@ import ( "bufio" "context" + "fmt" "io" "net/http" "net/url" @@ -11,27 +12,11 @@ "path/filepath" "strconv" "strings" - "sync" "time" "github.com/pkg/errors" ) -const ( - monday int = iota - tuesday - wednesday - thursday - friday -) - -//GetSyllabus is use main function struct. members using download html operation -type GetSyllabus struct { - year int - term string - outputdir string -} - // LectureDay include day of week (0~4, error -> 5), period, lastpriod (1~6) type LectureDay struct { DayOfWeek int @@ -43,10 +28,15 @@ type Lecture struct { ID string Name string - Day *LectureDay + Day LectureDay Teacher string } +type LectureWPath struct { + ID string + Path string +} + //CreateGetSyllabus is constructor and initialize from now time func CreateGetSyllabus() *GetSyllabus { var gs GetSyllabus @@ -85,66 +75,81 @@ return true, nil } -func (g *GetSyllabus) LecIDStoDonwlodSyllabus(ctx context.Context, lectureIDs []string) error { - var wg sync.WaitGroup +func (g *GetSyllabus) LecIDStoDonwlodSyllabus(ctx context.Context, lectureIDs []string) (*[]LectureWPath, error) { + //var wg sync.WaitGroup + ch := make(chan LectureWPath, len(lectureIDs)) for _, id := range lectureIDs { - wg.Add(1) + //wg.Add(1) go func(id string) { - defer wg.Done() - g.LecIDtoDownloadSyllabus(id) + //defer wg.Done() + outputPath, _ := g.LecIDtoDownloadSyllabus(id) + ch <- LectureWPath{ + ID: id, + Path: outputPath, + } }(id) } - wg.Wait() - return nil + //wg.Wait() + + var lwps []LectureWPath + for range lectureIDs { + lwps = append(lwps, <-ch) + } + return &lwps, nil } //LecIDtoDownloadSyllabus is download from lecture ID -func (g *GetSyllabus) LecIDtoDownloadSyllabus(lectureID string) error { +func (g *GetSyllabus) LecIDtoDownloadSyllabus(lectureID string) (string, error) { var strBuilder strings.Builder strBuilder.WriteString(lectureID) strBuilder.WriteString(".html") outputPath := filepath.Join(g.outputdir, strBuilder.String()) + if _, err := os.Stat(outputPath); err == nil { + return outputPath, nil + } + file, err := os.Create(outputPath) defer file.Close() if err != nil { - return errors.Wrap(err, "failed create html...") + return "", errors.Wrap(err, "failed create html...") } strBuilder.Reset() u, err := url.Parse(endpoint) if err != nil { - return err + return "", err } u.Path = path.Join(u.Path, "portal", "Public", "Syllabus", "SyllabusSearchStart.aspx") q := u.Query() - q.Set("lect_year", strconv.Itoa(g.year)) - q.Set("lect_cd", lectureID) + q.Set("lct_year", strconv.Itoa(g.year)) + q.Set("lct_cd", lectureID) q.Set("je_cd", "1") u.RawQuery = q.Encode() + fmt.Println(u.String()) res, err := http.Get(u.String()) defer res.Body.Close() if err != nil { - return errors.Wrap(err, "failed download html") + return "", errors.Wrap(err, "failed download html") } _, err = io.Copy(file, res.Body) if err != nil { - return errors.Wrap(err, "failed download html") + return "", errors.Wrap(err, "failed download html") } - return nil + return outputPath, nil } //LecIDwFilePath2LectureStruct is require LectureID (== Lecture.ID), filePath ( syllabus.html path) -func (g *GetSyllabus) LecIDwFilePath2LectureStruct(lectureID, filePath string) (*Lecture, error) { - file, err := os.Open(filePath) +func (g *GetSyllabus) LecIDwFilePath2LectureStruct(lwp *LectureWPath) (*Lecture, error) { + file, err := os.Open(lwp.Path) if err != nil { return nil, errors.Wrap(err, "failed open html file") @@ -152,7 +157,7 @@ scanner := bufio.NewScanner(file) var lec Lecture - lec.ID = lectureID + lec.ID = lwp.ID for scanner.Scan() { line := scanner.Text() @@ -162,15 +167,25 @@ if j := strings.Index(line, endSpan); j >= 0 { i += len(dayPeriodID) day := line[i:j] - lec.Day.DayOfWeek, err = kanjiday2int(day[0:dayOfWeeklen]) + + lec.Day.DayOfWeek = kanjiday2int(day[0:dayOfWeeklen]) if err != nil { return nil, errors.Wrap(err, "failed convert day") } lec.Day.Period, err = strconv.Atoi(day[dayOfWeeklen : dayOfWeeklen+1]) - - // dayの長さで〜があるかどうかが判定する + if err != nil { + return nil, errors.Wrap(err, "failed convert day") + } + if len(day) != (dayOfWeeklen + 1) { // dayOfWeeklen + 1 == 月3, 火2 + lec.Day.LastPeriod, err = strconv.Atoi(day[dayOfWeeklen+4:]) // 4 is \d + 〜 + if err != nil { + return nil, errors.Wrap(err, "failed convert day") + } + continue + } + lec.Day.LastPeriod = -1 + continue } - continue } // lecture name @@ -196,6 +211,18 @@ return &lec, nil } -func kanjiday2int(kanjiDay string) (int, error) { - return 0, nil +func kanjiday2int(kanjiDay string) int { + switch kanjiDay { + case "月": + return 0 + case "火": + return 1 + case "水": + return 2 + case "木": + return 3 + case "金": + return 4 + } + return -1 }