From c54ad0e501194ec5541a7630bf4205092576c303 Mon Sep 17 00:00:00 2001 From: fakelag <35497506+fakelag@users.noreply.github.com> Date: Sat, 3 Aug 2024 15:38:24 +0300 Subject: [PATCH] Add youtube search function --- youtubeapi/youtube.go | 227 ++++++++++++++++++++++++++----------- youtubeapi/youtube_test.go | 99 ++++++++++++++-- 2 files changed, 252 insertions(+), 74 deletions(-) diff --git a/youtubeapi/youtube.go b/youtubeapi/youtube.go index ac16bef..20fef86 100644 --- a/youtubeapi/youtube.go +++ b/youtubeapi/youtube.go @@ -3,6 +3,7 @@ package youtubeapi import ( "encoding/json" "errors" + "fmt" "math/rand" "net/url" "os/exec" @@ -80,6 +81,88 @@ func (yt *Youtube) SetCmdExecutor(exec cmd.CommandExecutor) { yt.executor = exec } +func (yt *Youtube) SearchYoutubeMedia(numSearchResults int, videoIdOrSearchTerm string) ([]*YoutubeMedia, error) { + ytDlp, err := getYtDlpPath() + + if err != nil { + return nil, err + } + + replacer := strings.NewReplacer( + "\"", "", + "'", "", + ) + + videoArg := fmt.Sprintf("ytsearch%d:%s", numSearchResults, replacer.Replace(videoIdOrSearchTerm)) + + args := []string{ + videoArg, + "--playlist-end", "1", + "--extract-audio", + "--quiet", + "--audio-format", "opus", + "--ignore-errors", + "--no-color", + "--no-check-formats", + "--max-downloads", "0", + "-s", + "--get-url", + "--print-json", + } + + resultChannel, errorChannel := yt.executor.RunCommandWithTimeout(ytDlp, yt.streamUrlTimeout, args...) + + var stdout string + + select { + case result := <-resultChannel: + stdout = *result + break + case err := <-errorChannel: + return nil, err + } + + searchResults := make([]*YoutubeMedia, 0) + + if len(stdout) == 0 { + return searchResults, nil + } + + jsonLines := strings.Split(stdout, "\n") + + if len(jsonLines) < 2 { + return searchResults, nil + } + + for i := 0; i < len(jsonLines); i += 2 { + videoStreamURL := jsonLines[i] + videoJson := jsonLines[i+1] + + var object YtDlpObject + if err := json.Unmarshal([]byte(videoJson), &object); err != nil { + return nil, err + } + + if object.Type != "video" { + continue + } + + media, _, err := yt.getMediaOrPlaylistFromJsonAndStreamURL(&object, videoJson, videoStreamURL) + + if err != nil { + return nil, err + } + + if media == nil { + continue + } + + searchResults = append(searchResults, media) + } + + return searchResults, nil +} + func (yt *Youtube) GetYoutubeMedia(videoIdOrSearchTerm string) (*YoutubeMedia, error) { ytDlp, err := getYtDlpPath() @@ -146,38 +229,12 @@ func (yt *Youtube) GetYoutubeMedia(videoIdOrSearchTerm string) (*YoutubeMedia, e return nil, err } - if object.Type == "video" { - var ytDlpVideo YtDlpVideo - if err := json.Unmarshal([]byte(videoJson), &ytDlpVideo); err != nil { - return nil, err - } - - media := &YoutubeMedia{ - ID: ytDlpVideo.ID, - VideoTitle: ytDlpVideo.Title, - VideoThumbnail: ytDlpVideo.Thumbnail, - VideoIsLiveStream: ytDlpVideo.IsLiveStream, - VideoDuration: time.Duration(ytDlpVideo.Duration) * time.Second, - VideoLink: "https://www.youtube.com/watch?v=" + ytDlpVideo.ID, - StreamURL: videoStreamURL, - ytAPI: yt, - } - - streamExpireUnixSecondsMatch := yt.streamUrlExpireRegex.FindStringSubmatch(videoStreamURL) - - if len(streamExpireUnixSecondsMatch) >= 4 { - unixSeconds, err := strconv.ParseInt(streamExpireUnixSecondsMatch[3], 10, 64) - - if err == nil && unixSeconds > 0 { - expirationTime := time.Unix(unixSeconds, 0) - media.StreamExpiresAt = &expirationTime - } - } - - return media, nil - } else { + if object.Type != "video" { return nil, ErrorUnrecognisedObject } + + media, _, err := yt.getMediaOrPlaylistFromJsonAndStreamURL(&object, videoJson, videoStreamURL) + return media, err } func (yt *Youtube) GetYoutubePlaylist(playlistIdOrUrl string) (*YoutubePlaylist, error) { @@ -229,40 +286,8 @@ func (yt *Youtube) GetYoutubePlaylist(playlistIdOrUrl string) (*YoutubePlaylist, return nil, ErrorUnrecognisedObject } - var ytDlpPlaylist YtDlpPlayList - if err := json.Unmarshal([]byte(playlistJson), &ytDlpPlaylist); err != nil { - return nil, err - } - - rngSource := rand.NewSource(time.Now().Unix()) - rng := rand.New(rngSource) - - playList := NewYoutubePlaylist(ytDlpPlaylist.ID, ytDlpPlaylist.Title, ytDlpPlaylist.PlaylistURL, rng, len(ytDlpPlaylist.Entries)) - - for index, video := range ytDlpPlaylist.Entries { - thumbnailUrl := "" - thumbnailWidth := 0 - - for _, thumbnail := range video.Thumbnails { - if thumbnail.Width > thumbnailWidth { - thumbnailUrl = thumbnail.URL - thumbnailWidth = thumbnail.Width - } - } - - playList.mediaList[index] = &YoutubeMedia{ - ID: video.ID, - VideoTitle: video.Title, - VideoThumbnail: thumbnailUrl, - VideoIsLiveStream: video.LiveStatus == "is_live", - VideoDuration: time.Duration(video.Duration) * time.Second, - VideoLink: "https://www.youtube.com/watch?v=" + video.ID, - StreamURL: "", - ytAPI: yt, - } - } - - return playList, nil + _, playList, err := yt.getMediaOrPlaylistFromJsonAndStreamURL(&object, playlistJson, "") + return playList, err } func NewYoutubePlaylist( @@ -312,6 +337,80 @@ func getYoutubeUrlVideoId(urlString string) string { return "" } +func (yt *Youtube) getMediaOrPlaylistFromJsonAndStreamURL( + object *YtDlpObject, + ytdlpJson string, + videoStreamURL string, +) (*YoutubeMedia, *YoutubePlaylist, error) { + if object.Type == "video" { + var ytDlpVideo YtDlpVideo + if err := json.Unmarshal([]byte(ytdlpJson), &ytDlpVideo); err != nil { + return nil, nil, err + } + + media := &YoutubeMedia{ + ID: ytDlpVideo.ID, + VideoTitle: ytDlpVideo.Title, + VideoThumbnail: ytDlpVideo.Thumbnail, + VideoIsLiveStream: ytDlpVideo.IsLiveStream, + VideoDuration: time.Duration(ytDlpVideo.Duration) * time.Second, + VideoLink: "https://www.youtube.com/watch?v=" + ytDlpVideo.ID, + StreamURL: videoStreamURL, + ytAPI: yt, + } + + streamExpireUnixSecondsMatch := yt.streamUrlExpireRegex.FindStringSubmatch(videoStreamURL) + + if len(streamExpireUnixSecondsMatch) >= 4 { + unixSeconds, err := strconv.ParseInt(streamExpireUnixSecondsMatch[3], 10, 64) + + if err == nil && unixSeconds > 0 { + expirationTime := time.Unix(unixSeconds, 0) + media.StreamExpiresAt = &expirationTime + } + } + + return media, nil, nil + } else if object.Type == "playlist" { + var ytDlpPlaylist YtDlpPlayList + if err := json.Unmarshal([]byte(ytdlpJson), &ytDlpPlaylist); err != nil { + return nil, nil, err + } + + rngSource := rand.NewSource(time.Now().Unix()) + rng := rand.New(rngSource) + + playList := NewYoutubePlaylist(ytDlpPlaylist.ID, ytDlpPlaylist.Title, ytDlpPlaylist.PlaylistURL, rng, len(ytDlpPlaylist.Entries)) + + for index, video := range ytDlpPlaylist.Entries { + thumbnailUrl := "" + thumbnailWidth := 0 + + for _, thumbnail := range video.Thumbnails { + if thumbnail.Width > thumbnailWidth { + thumbnailUrl = thumbnail.URL + thumbnailWidth = thumbnail.Width + } + } + + playList.mediaList[index] = &YoutubeMedia{ + ID: video.ID, + VideoTitle: video.Title, + VideoThumbnail: thumbnailUrl, + VideoIsLiveStream: video.LiveStatus == "is_live", + VideoDuration: time.Duration(video.Duration) * time.Second, + VideoLink: "https://www.youtube.com/watch?v=" + video.ID, + StreamURL: "", + ytAPI: yt, + } + } + + return nil, playList, nil + } else { + return nil, nil, ErrorUnrecognisedObject + } +} + func getYtDlpPath() (string, error) { path, err := exec.LookPath("yt-dlp") diff --git a/youtubeapi/youtube_test.go b/youtubeapi/youtube_test.go index 7ecf596..68fec9e 100644 --- a/youtubeapi/youtube_test.go +++ b/youtubeapi/youtube_test.go @@ -13,14 +13,16 @@ import ( . "github.com/onsi/gomega" ) -var mockVideoJson string = strings.ReplaceAll(`{ - "id": "123", - "fulltitle": "Mock Title", - "duration": 70, - "thumbnail": "foo", - "is_live": false, - "_type": "video" -}`, "\n", "") +func makeMockVideoJson(id string, title string) string { + return strings.ReplaceAll(fmt.Sprintf(`{ + "id": "%s", + "fulltitle": "%s", + "duration": 70, + "thumbnail": "foo", + "is_live": false, + "_type": "video" + }`, id, title), "\n", "") +} var mockPlaylistJson string = strings.ReplaceAll(`{ "id": "1234", @@ -78,7 +80,7 @@ var _ = Describe("YT Download", func() { When("Downloading a singular video", func() { It("Downloads a video stream URL from Youtube", func() { mockExecutor := &testutils.MockCommandExecutor{ - MockStdoutResult: "url123\n" + mockVideoJson, + MockStdoutResult: "url123\n" + makeMockVideoJson("123", "Mock Title"), } yt := youtubeapi.NewYoutubeAPI() @@ -106,7 +108,7 @@ var _ = Describe("YT Download", func() { "https://rr5---sn-qo5-ixas.googlevideo.com/videoplayback?expire=" + fmt.Sprintf("%d", timeUnix) + "&ei=123&foo=bar\n", } { mockExecutor := &testutils.MockCommandExecutor{ - MockStdoutResult: streamUrl + mockVideoJson, + MockStdoutResult: streamUrl + makeMockVideoJson("123", "Mock Title"), } yt := youtubeapi.NewYoutubeAPI() @@ -227,4 +229,81 @@ var _ = Describe("YT Download", func() { Expect(err).To(MatchError(youtubeapi.ErrorNoPlaylistFound)) }) }) + + When("Searching from youtube", func() { + It("Downloads a video stream URL from Youtube", func() { + lines := []string{ + "url4video1", + makeMockVideoJson("1", "Video 1"), + "url4video2", + makeMockVideoJson("2", "Video 2"), + "url4video3", + makeMockVideoJson("3", "Video 3"), + } + mockExecutor := &testutils.MockCommandExecutor{ + MockStdoutResult: strings.Join(lines, "\n"), + } + + yt := youtubeapi.NewYoutubeAPI() + yt.SetCmdExecutor(mockExecutor) + + searchResults, err := yt.SearchYoutubeMedia(3, "foo") + + Expect(err).To(BeNil()) + Expect(searchResults).NotTo(BeNil()) + Expect(searchResults).To(HaveLen(3)) + + vid := searchResults[0] + Expect(vid).NotTo(BeNil()) + Expect(vid.ID).To(Equal("1")) + Expect(vid.Title()).To(Equal("Video 1")) + Expect(vid.FileURL()).To(Equal("url4video1")) + Expect(vid.Thumbnail()).To(Equal("foo")) + + vid = searchResults[1] + Expect(vid).NotTo(BeNil()) + Expect(vid.ID).To(Equal("2")) + Expect(vid.Title()).To(Equal("Video 2")) + Expect(vid.FileURL()).To(Equal("url4video2")) + Expect(vid.Thumbnail()).To(Equal("foo")) + + vid = searchResults[2] + Expect(vid).NotTo(BeNil()) + Expect(vid.ID).To(Equal("3")) + Expect(vid.Title()).To(Equal("Video 3")) + Expect(vid.FileURL()).To(Equal("url4video3")) + Expect(vid.Thumbnail()).To(Equal("foo")) + }) + + It("Returns sensible results when receiving an invalid response from ytdlp", func() { + mockExecutor := &testutils.MockCommandExecutor{ + MockStdoutResult: "streamurl\n{", + } + + yt := youtubeapi.NewYoutubeAPI() + yt.SetCmdExecutor(mockExecutor) + + searchResults, err := yt.SearchYoutubeMedia(5, "foo") + Expect(err).To(MatchError("unexpected end of JSON input")) + Expect(searchResults).To(BeNil()) + + mockExecutor.MockStdoutResult = "streamurl\n{\"_type\":\"something\"}" + + searchResults, err = yt.SearchYoutubeMedia(5, "foo") + Expect(err).To(BeNil()) + Expect(searchResults).To(HaveLen(0)) + + mockExecutor.MockStdoutResult = "streamurl" + + searchResults, err = yt.SearchYoutubeMedia(5, "foo") + Expect(err).To(BeNil()) + Expect(searchResults).To(HaveLen(0)) + + mockExecutor.MockStdoutResult = "" + + searchResults, err = yt.SearchYoutubeMedia(5, "foo") + Expect(err).To(BeNil()) + Expect(searchResults).To(HaveLen(0)) + }) + }) })