Sync live notes list

Signed-off-by: Yuan Si <do4suki@gmail.com>
2023-03-29 08:41:51 +08:00
parent c6eaae9ae1
commit f5c85e5a10
7 changed files with 321 additions and 186 deletions
@@ -7,7 +7,15 @@ require (
 	github.com/redis/go-redis/v9 v9.0.2
 )

-require github.com/mattn/go-sqlite3 v1.14.16
+require (
+	github.com/mattn/go-sqlite3 v1.14.16
+	github.com/tidwall/gjson v1.14.4
+)
+
+require (
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+)

 require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -68,6 +68,12 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
+github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=
 github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
@@ -5,7 +5,7 @@ import (
 	"honoka-chan/handler"
 	"honoka-chan/middleware"
 	"honoka-chan/sifcap"
-	"honoka-chan/tools"
+	_ "honoka-chan/tools"
 	"honoka-chan/xclog"

 	"github.com/gin-gonic/gin"
@@ -16,11 +16,6 @@ func init() {
 }

 func main() {
-	tools.AnalysisApi1Data("assets/api1.json")
-	tools.AnalysisApi2Data("assets/api2.json")
-	tools.AnalysisApi3Data("assets/api3.json")
-	// tools.ListUnitData()
-
 	if config.Conf.SifCap.Enabled {
 		sifcap.Start()
 	} else {
@@ -0,0 +1,182 @@
+package tools
+
+import (
+	"encoding/json"
+	"honoka-chan/database"
+	"honoka-chan/model"
+	"honoka-chan/utils"
+)
+
+func AnalysisApi1Data(path string) {
+	apiData := utils.ReadAllText(path)
+	if apiData != "" {
+		var obj model.Response
+		err := json.Unmarshal([]byte(apiData), &obj)
+		if err != nil {
+			panic(err)
+		}
+
+		var data interface{}
+		err = json.Unmarshal(obj.ResponseData, &data)
+		if err != nil {
+			panic(err)
+		}
+		// resultType := reflect.TypeOf(data)
+		// if resultType.Kind() == reflect.Map {
+		// 	data = data.(map[string]interface{})
+		// }
+		result := data.([]interface{})
+		for k, v := range result {
+			m, err := json.Marshal(v)
+			if err != nil {
+				panic(err)
+			}
+
+			key := ""
+			switch k {
+			case 0:
+				key = "live_status_result"
+			case 1:
+				key = "live_list_result"
+			case 2:
+				key = "unit_list_result"
+			case 3:
+				key = "unit_deck_result"
+			case 4:
+				key = "unit_support_result"
+			case 5:
+				key = "owning_equip_result"
+			case 6:
+				key = "costume_list_result"
+			case 7:
+				key = "album_unit_result"
+			case 8:
+				key = "scenario_status_result"
+			case 9:
+				key = "subscenario_status_result"
+			case 10:
+				key = "event_scenario_result"
+			case 11:
+				key = "multi_unit_scenario_result"
+			case 12:
+				key = "product_result"
+			case 13:
+				key = "banner_result"
+			case 14:
+				key = "item_marquee_result"
+			case 15:
+				key = "user_intro_result"
+			case 16:
+				key = "special_cutin_result"
+			case 17:
+				key = "award_result"
+			case 18:
+				key = "background_result"
+			case 19:
+				key = "stamp_result"
+			case 20:
+				key = "exchange_point_result"
+			case 21:
+				key = "live_se_result"
+			case 22:
+				key = "live_icon_result"
+			case 23:
+				key = "item_list_result"
+			case 24:
+				key = "marathon_result"
+			case 25:
+				key = "challenge_result"
+			}
+
+			if key != "" {
+				database.RedisCli.HSet(database.RedisCtx, "temp_dataset", key, string(m))
+			}
+		}
+	}
+}
+
+func AnalysisApi2Data(path string) {
+	apiData := utils.ReadAllText(path)
+	if apiData != "" {
+		var obj model.Response
+		err := json.Unmarshal([]byte(apiData), &obj)
+		if err != nil {
+			panic(err)
+		}
+
+		var data interface{}
+		err = json.Unmarshal(obj.ResponseData, &data)
+		if err != nil {
+			panic(err)
+		}
+		// resultType := reflect.TypeOf(data)
+		// if resultType.Kind() == reflect.Map {
+		// 	data = data.(map[string]interface{})
+		// }
+		result := data.([]interface{})
+		for k, v := range result {
+			m, err := json.Marshal(v)
+			if err != nil {
+				panic(err)
+			}
+
+			key := ""
+			switch k {
+			case 0:
+				key = "login_topinfo_result"
+			case 1:
+				key = "login_topinfo_once_result"
+			case 2:
+				key = "unit_accessory_result"
+			case 3:
+				key = "museum_result"
+			}
+
+			if key != "" {
+				database.RedisCli.HSet(database.RedisCtx, "temp_dataset", key, string(m))
+			}
+		}
+	}
+}
+
+func AnalysisApi3Data(path string) {
+	apiData := utils.ReadAllText(path)
+	if apiData != "" {
+		var obj model.Response
+		err := json.Unmarshal([]byte(apiData), &obj)
+		if err != nil {
+			panic(err)
+		}
+
+		var data interface{}
+		err = json.Unmarshal(obj.ResponseData, &data)
+		if err != nil {
+			panic(err)
+		}
+		// resultType := reflect.TypeOf(data)
+		// if resultType.Kind() == reflect.Map {
+		// 	data = data.(map[string]interface{})
+		// }
+		result := data.([]interface{})
+		for k, v := range result {
+			m, err := json.Marshal(v)
+			if err != nil {
+				panic(err)
+			}
+
+			key := ""
+			switch k {
+			case 0:
+				key = "profile_livecnt_result"
+			case 1:
+				key = "profile_card_ranking_result"
+			case 2:
+				key = "profile_info_result"
+			}
+
+			if key != "" {
+				database.RedisCli.HSet(database.RedisCtx, "temp_dataset", key, string(m))
+			}
+		}
+	}
+}
@@ -0,0 +1,117 @@
+package tools
+
+import (
+	"database/sql"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/tidwall/gjson"
+)
+
+func SyncNotesList() {
+	db, err := sql.Open("sqlite3", "assets/live.db")
+	if err != nil {
+		panic(err)
+	}
+	defer func() {
+		db.Close()
+		fmt.Println("Sync notes list done!")
+	}()
+	db.SetMaxOpenConns(1)
+
+	sql := `SELECT live_setting_id,notes_setting_asset,notes_list FROM live_setting_m ORDER BY live_setting_id ASC`
+	rows, err := db.Query(sql)
+	if err != nil {
+		panic(err)
+	}
+
+	liveList := make(map[int]string)
+
+	for rows.Next() {
+		var id int
+		var asset string
+		var notes interface{}
+		err = rows.Scan(&id, &asset, &notes)
+		if err != nil {
+			fmt.Println(err)
+			continue
+		}
+
+		if notes != nil {
+			continue
+		}
+
+		liveList[id] = asset
+	}
+	rows.Close()
+
+	// fmt.Println(liveList)
+
+	for id, asset := range liveList {
+		url := "https://card.niconi.co.ni/live/" + asset
+		fmt.Println(url)
+
+		res, err := http.Get(url)
+		if err != nil {
+			fmt.Println(err)
+			continue
+		}
+		if res.StatusCode == http.StatusNotFound {
+			fmt.Println(res.StatusCode)
+			continue
+		}
+		body, err := io.ReadAll(res.Body)
+		if err != nil {
+			fmt.Println(err)
+			continue
+		}
+		_ = res.Body.Close()
+		// fmt.Println(string(body))
+
+		regex := regexp.MustCompile("var lives = (.*?)\n")
+		matchs := regex.FindSubmatch(body)
+		match := string(matchs[len(matchs)-1])
+
+		match = strings.ReplaceAll(match, "{\\\"", "{\"")
+		match = strings.ReplaceAll(match, "\\\":", "\":")
+		match = strings.ReplaceAll(match, ",\\\"", ",\"")
+		match = strings.ReplaceAll(match, "notes_list\":\"", "notes_list\":")
+		match = strings.ReplaceAll(match, "]\"}]", "]}]")
+
+		var notesList string
+		gjson.Parse(match).ForEach(func(key, value gjson.Result) bool {
+			notesList = value.Get("notes_list").String()
+			return true
+		})
+		// fmt.Println(notesList)
+
+		if notesList == "" {
+			fmt.Println("notes_list is null")
+			continue
+		}
+
+		stmt, err := db.Prepare("UPDATE live_setting_m SET notes_list = ? WHERE live_setting_id = ?")
+		if err != nil {
+			fmt.Println(err)
+			continue
+		}
+		ret, err := stmt.Exec(notesList, id)
+		if err != nil {
+			fmt.Println(err)
+			continue
+		}
+		r, err := ret.RowsAffected()
+		if err != nil {
+			fmt.Println(err)
+			continue
+		}
+		fmt.Println("RowsAffected:", r)
+
+		time.Sleep(time.Second)
+	}
+}
@@ -1,182 +1,9 @@
 package tools

-import (
-	"encoding/json"
-	"honoka-chan/database"
-	"honoka-chan/model"
-	"honoka-chan/utils"
-)
-
-func AnalysisApi1Data(path string) {
-	apiData := utils.ReadAllText(path)
-	if apiData != "" {
-		var obj model.Response
-		err := json.Unmarshal([]byte(apiData), &obj)
-		if err != nil {
-			panic(err)
-		}
-
-		var data interface{}
-		err = json.Unmarshal(obj.ResponseData, &data)
-		if err != nil {
-			panic(err)
-		}
-		// resultType := reflect.TypeOf(data)
-		// if resultType.Kind() == reflect.Map {
-		// 	data = data.(map[string]interface{})
-		// }
-		result := data.([]interface{})
-		for k, v := range result {
-			m, err := json.Marshal(v)
-			if err != nil {
-				panic(err)
-			}
-
-			key := ""
-			switch k {
-			case 0:
-				key = "live_status_result"
-			case 1:
-				key = "live_list_result"
-			case 2:
-				key = "unit_list_result"
-			case 3:
-				key = "unit_deck_result"
-			case 4:
-				key = "unit_support_result"
-			case 5:
-				key = "owning_equip_result"
-			case 6:
-				key = "costume_list_result"
-			case 7:
-				key = "album_unit_result"
-			case 8:
-				key = "scenario_status_result"
-			case 9:
-				key = "subscenario_status_result"
-			case 10:
-				key = "event_scenario_result"
-			case 11:
-				key = "multi_unit_scenario_result"
-			case 12:
-				key = "product_result"
-			case 13:
-				key = "banner_result"
-			case 14:
-				key = "item_marquee_result"
-			case 15:
-				key = "user_intro_result"
-			case 16:
-				key = "special_cutin_result"
-			case 17:
-				key = "award_result"
-			case 18:
-				key = "background_result"
-			case 19:
-				key = "stamp_result"
-			case 20:
-				key = "exchange_point_result"
-			case 21:
-				key = "live_se_result"
-			case 22:
-				key = "live_icon_result"
-			case 23:
-				key = "item_list_result"
-			case 24:
-				key = "marathon_result"
-			case 25:
-				key = "challenge_result"
-			}
-
-			if key != "" {
-				database.RedisCli.HSet(database.RedisCtx, "temp_dataset", key, string(m))
-			}
-		}
-	}
-}
-
-func AnalysisApi2Data(path string) {
-	apiData := utils.ReadAllText(path)
-	if apiData != "" {
-		var obj model.Response
-		err := json.Unmarshal([]byte(apiData), &obj)
-		if err != nil {
-			panic(err)
-		}
-
-		var data interface{}
-		err = json.Unmarshal(obj.ResponseData, &data)
-		if err != nil {
-			panic(err)
-		}
-		// resultType := reflect.TypeOf(data)
-		// if resultType.Kind() == reflect.Map {
-		// 	data = data.(map[string]interface{})
-		// }
-		result := data.([]interface{})
-		for k, v := range result {
-			m, err := json.Marshal(v)
-			if err != nil {
-				panic(err)
-			}
-
-			key := ""
-			switch k {
-			case 0:
-				key = "login_topinfo_result"
-			case 1:
-				key = "login_topinfo_once_result"
-			case 2:
-				key = "unit_accessory_result"
-			case 3:
-				key = "museum_result"
-			}
-
-			if key != "" {
-				database.RedisCli.HSet(database.RedisCtx, "temp_dataset", key, string(m))
-			}
-		}
-	}
-}
-
-func AnalysisApi3Data(path string) {
-	apiData := utils.ReadAllText(path)
-	if apiData != "" {
-		var obj model.Response
-		err := json.Unmarshal([]byte(apiData), &obj)
-		if err != nil {
-			panic(err)
-		}
-
-		var data interface{}
-		err = json.Unmarshal(obj.ResponseData, &data)
-		if err != nil {
-			panic(err)
-		}
-		// resultType := reflect.TypeOf(data)
-		// if resultType.Kind() == reflect.Map {
-		// 	data = data.(map[string]interface{})
-		// }
-		result := data.([]interface{})
-		for k, v := range result {
-			m, err := json.Marshal(v)
-			if err != nil {
-				panic(err)
-			}
-
-			key := ""
-			switch k {
-			case 0:
-				key = "profile_livecnt_result"
-			case 1:
-				key = "profile_card_ranking_result"
-			case 2:
-				key = "profile_info_result"
-			}
-
-			if key != "" {
-				database.RedisCli.HSet(database.RedisCtx, "temp_dataset", key, string(m))
-			}
-		}
-	}
+func init() {
+	AnalysisApi1Data("assets/api1.json")
+	AnalysisApi2Data("assets/api2.json")
+	AnalysisApi3Data("assets/api3.json")
+	// ListUnitData()
+	go SyncNotesList()
 }