Merge pull request #14 from herowinb/dev-twitch

Twitch.tv site support
    Can monitor, notification (Discord/Telegram/Slack/Email) and download if channel live
    Not support: Keyword Filter, Member only stream, Audio only

README.md

@@ -2,16 +2,19 @@
 
 # `New feature`
  * [Run on Docker](#how-run-on-synology-nas-docker)
+ * Site support:
+   * Youtube.com (and youtu.be)
+   * Twitch.tv
  * Metadata write
  * Archive live chat. Supported member-only stream. 2 modes:
    * Simple: Txt file, easy to read. You can change format in `/config/chat-format.json`
    * Full: Json file, playable via  https://archive.ragtag.moe/player
- * Keyword filter
- * [Member only (channel and video) support ](#how-monitor-member-only-stream)
- * [Add option Long interval loop](#how-use-long-interval)
+ * Keyword filter (Not apply to Twitch)
+ * [Member only (channel and video) support ](#how-monitor-member-only-stream) (Not apply to Twitch)
+ * [Add option Long interval loop](#how-use-long-interval) (Not apply to Twitch)
  * Notify (CLI, Discord and Email) when new live-dl updates are available
  * [Discord notification](#how-use-Discord-notification)
- * Audio only mode
+ * Audio only mode (Not apply to Twitch)
  * No log mode
  * Use [YTARCHIVE.py](https://github.com/Kethsar/ytarchive) to download Youtube Live Stream. If you not change config, Live-dl still use default Youtube-dl
 

config/auto.sh

@@ -17,3 +17,7 @@ sleep 20
 sleep 15
 
 #nohup /bin/bash /usr/src/app/live-dl -kf --filter Unarchive|Karaoke|Sing -ao  https://www.youtube.com/channel/UCP0BspO_AMEe3aQqqpo89Dg &>/youtube-dl/live-dl-moona-audio-only.$(date +%d%b%y-%H%M%S).log &
+
+sleep 15
+
+#nohup /bin/bash /usr/src/app/live-dl -i 60 -ac https://www.twitch.tv/*** &>/youtube-dl/live-dl-***.$(date +%d%b%y-%H%M%S).log &

config/config.example.yml

@@ -91,6 +91,7 @@ map:
     bilibili: https://live.bilibili.com/14917277
     telegram: -1001452928607
     tiktok: minatoaqua
+    twitch: https://www.twitch.tv/minatoaquach_hololive/
 
   - name: 神楽めあ
     mode: monitor

live-dl

@@ -271,6 +271,11 @@ function func_create_url() {
       __info "$(__red "Non-supported YouTube URL")"
       exit 1
     fi
+  elif [[ "$_url" =~ "twitch.tv" ]]; then
+    __info "Twitch detected"
+    DL_URL="$_url"
+    DL_PLATFORM="Twitch"
+    DL_TYPE="Twitch"
 
   elif [[ "$_url" =~ "live.bilibili.com" ]]; then
     __info "Bilibili detected"
@@ -715,6 +720,120 @@ function func_check_state() {
   fi
 }
 
+# Clone of func_check_state but for Twitch
+function func_check_state_twitch() {
+  local _url=$1
+  local _mode=$2
+
+  __debug "start func_check_state_twitch"
+
+  CONTENT_STATE="invalid"
+
+  local address_param=$(func_gen_curl_address_param)
+  if [ "$address_param" ]; then
+    __info "using address: ${address_param##* }"
+  fi
+
+  local _body=$(curl -s --compressed -H "User-Agent: $USER_AGENT" $address_param -b "$COOKIES_FILE" -s "$_url")
+
+  if [[ "$_body" == *"isLiveBroadcast"* ]]; then
+
+    METADATA_CURL=`echo "$_body" | sed 's/^.*application\/ld+json">//' | sed 's/<\/script.*$//' | jq -r . `
+
+    if [ ! -z "$METADATA_CURL" ]; then
+      __debug "Got valid cURL metadata"
+
+      IS_LIVE=$(echo $METADATA_CURL | jq -r '.[].publication.isLiveBroadcast | select(.!=null)')
+      __debug "Is live: $IS_LIVE"
+      UPLOADER=$(echo "$METADATA_CURL" | jq -r '.[].name')
+    fi
+
+    if [ "$IS_LIVE" == "true" ]; then
+      CONTENT_STATE="live"
+      __info "cURL check seems goes live now, continue checking..."
+    fi
+
+    if ([ "$CONTENT_STATE" == "live" ] && [ "$_mode" != "lazy" ]); then
+      __info "Re-checking via youtube-dl..."
+
+      local _youtube_dl_address_param=$(func_gen_youtube_dl_address_param)
+      if [ "$_youtube_dl_address_param" ]; then
+        __info "using address: ${_youtube_dl_address_param##* }"
+      fi
+      METADATA=`youtube-dl --ignore-config --no-playlist --playlist-items 0 --cookies "$COOKIES_FILE" \
+        --skip-download --print-json --referer 'https://www.twitch.tv/directory' \
+        -o '%(upload_date)s %(title)s (%(id)s).%(ext)s' \
+        ${_youtube_dl_address_param} \
+        "$_url" 2>&1`
+
+      if [ ! -z "$METADATA" ]; then
+        __debug "Got valid youtube-dl metadata"
+
+        if [[ "$METADATA" == '{'* ]]; then
+          # use 2>/dev/null to ignore errors line when use jq
+          local _is_live=`echo "$METADATA" | jq '.is_live' 2>/dev/null`
+
+          VIDEO_ID=$(echo "$METADATA" | jq -r '.id' 2>/dev/null)
+          FULLTITLE=$(echo "$METADATA" | jq -r '.description' 2>/dev/null) # Default .fulltitle is Twitchusername (live) date time. So don't use it
+          DESCRIPTION=$(echo "$METADATA" | jq -r '.description' 2>/dev/null)
+          UPLOAD_DATE=$(echo "$METADATA" | jq -r '.upload_date' 2>/dev/null)
+          UPLOADER=$(echo "$METADATA" | jq -r '.uploader' 2>/dev/null)
+          CHANNEL_ID=$(echo "$METADATA" | jq -r '.webpage_url' 2>/dev/null)
+          THUMBNAIL=$(echo "$METADATA" | jq -r '.thumbnail' 2>/dev/null)
+          WEBPAGE_URL=$(echo "$METADATA" | jq -r '.webpage_url' 2>/dev/null)
+          CHANNEL_URL=$(echo "$METADATA" | jq -r '.webpage_url' 2>/dev/null)
+
+          if [ $REMOVE_TITLE == "true" ]; then
+              __info "Running remove title function..."
+              FILENAME=$(echo "$UPLOAD_DATE ($VIDEO_ID)")
+          else
+              FULLTITLE=`escape_filename "${FULLTITLE}"`
+              FILENAME=$(echo "$UPLOAD_DATE $FULLTITLE ($VIDEO_ID)")
+          fi
+
+          func_process_channel_mapping
+
+          func_finalize_vars
+
+          if [ "$_is_live" == "true" ]; then
+            __debug "Got state: live";
+            CONTENT_STATE="live"
+          else
+            __info "youtube-dl does not returns a valid state";
+            CONTENT_STATE="invalid"
+          fi
+        else
+          __debug "$METADATA"
+          if [[ "$METADATA" =~ "This video is unavailable" ]]; then
+            # Suppress `video is unavailable` errors since they're common when stream is not live
+            __debug "This video is unavailable, maybe not live at the moment."
+            CONTENT_STATE="unavailable"
+          elif [[ "$METADATA" =~ "HTTP Error 404" ]]; then
+            __info "$(__yellow "Not a valid video page (Error 404)!")"
+            CONTENT_STATE="not_found"
+          elif [[ "$METADATA" =~ "HTTP Error 429" ]]; then
+            __info "$(__yellow "Your IP is limited by YouTube (Error 429)! Check here: https://github.com/herowinb/live-dl/issues/3 ")"
+            CONTENT_STATE="too_many_requests"
+          elif [[ "$METADATA" =~ "is offline" ]]; then
+            __debug "Stream already offline or maybe not live at the moment."
+            CONTENT_STATE="offline"
+          else
+            __info "$(__yellow "Unknown metadata")"
+            __info "$METADATA"
+            CONTENT_STATE="invalid"
+          fi
+        fi
+      else
+        __info "$(__yellow "Second check failed, not valid youtube-dl metadata")"
+        CONTENT_STATE="invalid"
+      fi
+
+      # Print current state for second check
+      __debug "Current state for second check: $(__yellow "$CONTENT_STATE")"
+    fi
+  fi
+}
+
 function func_send_email() {
   local _title=$1
   local _content=$2
@@ -1165,8 +1284,13 @@ function func_process_channel_mapping() {
   # https://stackoverflow.com/a/40027637/412385
 
   if [ "$CHANNEL_ID" ]; then
-    UPLOADER_MAPPED_RAW=`echo "$CONFIG_RAW" | jq -r --arg CHANNEL_ID "$CHANNEL_ID" \
-      '.map[] | select(.youtube | contains($CHANNEL_ID))'`
+    if [[ $DL_PLATFORM == "Twitch" ]]; then
+      UPLOADER_MAPPED_RAW=`echo "$CONFIG_RAW" | jq -r --arg CHANNEL_ID "$CHANNEL_ID" \
+        '.map[] | select(.twitch==$CHANNEL_ID)'`
+    else
+      UPLOADER_MAPPED_RAW=`echo "$CONFIG_RAW" | jq -r --arg CHANNEL_ID "$CHANNEL_ID" \
+        '.map[] | select(.youtube | contains($CHANNEL_ID))'`
+    fi
 
     # Check if current uploader has custom name mapping in config
     UPLOADER_MAPPED=`echo "$UPLOADER_MAPPED_RAW" | jq -r .name`
@@ -1248,13 +1372,17 @@ function func_finalize_vars() {
     fi
   fi
 
+#  if [[ $DL_PLATFORM == "Twitch" ]]; then
+#    OUTPUT_BASE="$BASE_DIR/$FILENAME_BASE/twitch"
+#  fi
+
   # Full output dir
   FILENAME=$(func_create_file_name "$FILENAME" "$OUTPUT_BASE")
   OUTPUT_PATH="$OUTPUT_BASE/$FILENAME"
 
   # Write prettified JSON metadata to file
   func_prepare_dir "$OUTPUT_BASE"
-  METADATA_PRETTIFIED=`echo $METADATA | jq .`
+  METADATA_PRETTIFIED=`echo $METADATA | jq . 2>/dev/null`
   echo "$METADATA_PRETTIFIED" > "$OUTPUT_PATH.json"
 
   # Select logs location
@@ -1397,6 +1525,27 @@ function func_download_youtube() {
   fi
 }
 
+function func_download_twitch() {
+  __info "Start downloading $FILENAME"
+
+  # Log tail inspired by https://github.com/printempw/live-stream-recorder
+  __info "Logging: tail -f \"$LOG_PATH\""
+
+  if [ "$CONTENT_STATE" == "live" ]; then
+  	youtube-dl \
+  		--ignore-config \
+      --hls-prefer-native \
+  		--hls-use-mpegts \
+      --no-part \
+  		--fragment-retries 30 \
+      --retries 60 \
+      $FORTMAT_OPTION \
+  		--cookies "$COOKIES_FILE" \
+  		-o "${OUTPUT_PATH}${EXTENSION}" \
+  		"${WEBPAGE_URL}" >> "$LOG_PATH" 2>&1
+    fi
+}
+
 function func_download_bilibili() {
   __info "bilibili download func"
 }
@@ -1604,6 +1753,10 @@ if ! [ -f ./config/config.yml ]; then
   exit 1;
 else
   CONFIG_RAW=`cat ./config/config.yml | yq -rR .`
+  if [[ -z $CONFIG_RAW ]]; then
+    __info "$(__red "Wrong config.yml format, make sure you have configured the correct format.")"
+    exit 1;
+  fi
 fi
 
 # Set default settings
@@ -1728,9 +1881,15 @@ while true; do
     # Check if url types and live available
     # After creating URL with func_create_url, I need to further process the URL to determine
     # the type (video, live stream) and state (is live or not ) of the given URL
-    func_check_state "$DL_URL"
+
+    if [ "$DL_PLATFORM" == "Twitch" ]; then
+      func_check_state_twitch "$DL_URL"
+    else
+      func_check_state "$DL_URL"
+    fi
 
     INTERVAL2=$INTERVAL
+    if [ "$DL_PLATFORM" == "YouTube" ]; then
       # Get the time now and compare with upcoming time
       # If upcoming time longer than LONG_INTERVAL, live-dl will use LONG_INTERVAL
       # If upcoming time less than LONG_INTERVAL, live-dl will use INTERVAL
@@ -1746,6 +1905,7 @@ while true; do
           INTERVAL2=$LONG_INTERVAL
         fi
       fi
+    fi
 
     if [ "$CONTENT_STATE" == "live" ]; then
       break
@@ -1769,14 +1929,14 @@ while true; do
   #
   # Platform: YouTube
   #
-  if [ "$DL_PLATFORM" == "YouTube" ]; then
+  if [ "$DL_PLATFORM" == "YouTube" ] || [ "$DL_PLATFORM" == "Twitch" ]; then
     # Print summary
     echo -e "\n$SUMMARY"
 
     #
     # Type: YouTube channel
     #
-    if [ "$DL_TYPE" == "channel" ]; then
+    if [ "$DL_TYPE" == "channel" ] || [ "$DL_TYPE" == "Twitch" ]; then
 
       #
       # Mode: download
@@ -1813,7 +1973,11 @@ while true; do
         __info "Notifier mode, no video will be downloaded, continue monitoring..."
 
         while true; do
-          func_check_state "$DL_URL" "lazy"
+          if [ "$DL_PLATFORM" == "Twitch" ]; then
+            func_check_state_twitch "$DL_URL" "lazy"
+          else
+            func_check_state "$DL_URL" "lazy"
+          fi
 
           if [ "$CONTENT_STATE" != "live" ]; then
             __info "Live stopped"