Merge pull request #950 from alpgul/newFeature-webParserWithPattern

New feature web parser with pattern

Author: phunkyfish

CMakeLists.txt

@@ -45,7 +45,10 @@ set(IPTV_SOURCES src/addon.cpp
                  src/iptvsimple/utilities/Logger.cpp
                  src/iptvsimple/utilities/SettingsMigration.cpp
                  src/iptvsimple/utilities/StreamUtils.cpp
-                 src/iptvsimple/utilities/WebUtils.cpp)
+                 src/iptvsimple/utilities/WebUtils.cpp
+                 src/iptvsimple/utilities/WebStreamExtractor.cpp
+                 src/iptvsimple/utilities/CurlUtils.cpp
+                 )
 
 set(IPTV_HEADERS src/addon.h
                  src/IptvSimple.h
@@ -76,7 +79,10 @@ set(IPTV_HEADERS src/addon.h
                  src/iptvsimple/utilities/StreamUtils.h
                  src/iptvsimple/utilities/TimeUtils.h
                  src/iptvsimple/utilities/WebUtils.h
-                 src/iptvsimple/utilities/XMLUtils.h)
+                 src/iptvsimple/utilities/XMLUtils.h
+                 src/iptvsimple/utilities/WebStreamExtractor.h
+                 src/iptvsimple/utilities/CurlUtils.h
+                 )
 
 addon_version(pvr.iptvsimple IPTV)
 add_definitions(-DIPTV_VERSION=${IPTV_VERSION})

README.md

@@ -408,7 +408,7 @@ Here’s some examples of how the different formats would look:
 #### M3U format elements:
 
 ```
-#EXTM3U tvg-shift="-4.5" x-tvg-url="http://path-to-xmltv/guide.xml catchup-correction="-2.5"
+#EXTM3U tvg-shift="-4.5" x-tvg-url="http://path-to-xmltv/guide.xml" catchup-correction="-2.5"
 #EXTINF:0 tvg-id="channel-x" tvg-name="Channel_X" group-title="Entertainment" tvg-chno="10" tvg-logo="http://path-to-icons/channel-x.png" radio="true" tvg-shift="-3.5",Channel X
 #EXTVLCOPT:program=745
 #KODIPROP:key=val
@@ -451,6 +451,10 @@ http://path-to-stream/live/channel-m.mkv
 http://path-to-stream/live/channel-n.mkv
 #EXTINF:-1 media-dir="/movies/scifi",Channel O
 http://path-to-stream/live/channel-o.mkv
+#EXTINF:-1 tvg-name="Channel-P", Channel-P
+#WEBPROP:web-regex="([^"]+\.m3u8)"
+#WEBPROP:web-headers=user-agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36&referer:https://google.com/
+@http://path-to-stream/live/channel-p.html|user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36
 ```
 
 *Explanation for Catchup entries*
@@ -528,8 +532,12 @@ http://path-to-stream/live/channel-z.ts
 - `#EXTGRP`: A semi-colon separted list of channel groups. Note that this is a begin directive, i.e. all channels following this directive will have these groups until an empty `#EXTGRP` directive is reached. These groupings wil also be reset by any `group-title` tag for an `#EXTINF` channel directive.
 - `#KODIPROP`: A single property in the format `key=value` that can be passed to Kodi. Multiple can be passed each on a separate line.
 - `#EXTVLCOPT`: A single property in the format `key=value` that can be passed to Kodi. Multiple can be passed each on a separate line. Note that if either a `http-user-agent` or a `http-referrer` property is found it will added to the URL as a HTTP header as `user-agent` or `referrer` respectively if not already provided in the URL. These two fields specifically will be dropped as properties whether or not they are added as header values. They will be added in the same format as the `URL` below.
+- `#WEBPROP`: Properties used for web scraping streams from HTML pages. Multiple can be passed each on a separate line.
+  - `web-regex`: A regular expression pattern to extract the stream URL from the HTML content. The first capture group will be used as the stream URL. The URL must be captured within parentheses in the regex pattern, e.g. "([^"]+\.m3u8)".
+  - `web-headers`: HTTP headers to be used when making the request to the web page. Format is `name1:value1&name2:value2`.
 - `#EXT-X-PLAYLIST-TYPE`: If this element is present with a value of `VOD` (Video on Demand) the stream is marked as not being live.
 - `URL`: The final line in each channel stanza is the URL used for the stream. Appending `|user-agent=<agent-name>` will change the user agent. Other HTTP header fields can be set in the same fashion: `|name1=val1&name2=val2` etc. The header fields supported in this way by Kodi can be found [here](#http-header-fields-supported-by-kodi). If you want to pass custom headers that are not supported by Kodi you need to prefix them with an `!`, for example: : `|!name1=val1&!name2=val2`.
+  - `@`: When a URL starts with '@' symbol (e.g. '@http://path-to-stream/live/channel-p.html'), it indicates that the URL points to a web page that contains the actual stream URL. The addon will attempt to extract the stream URL from this web page using either the provided `web-regex` pattern or default patterns. This is useful when the actual stream URL is embedded within a web page rather than being directly accessible.
 
 When processing an XMLTV file the addon will attempt to find a channel loaded from the M3U that matches the EPG channel. It will cycle through the full set of M3U channels checking for one condition on each pass. The first channel found to match is the channel chosen for this EPG channel data.
 

depends/common/zlib/01-build-static.patch

@@ -1,33 +1,35 @@
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -147,10 +147,11 @@
+@@ -149,12 +149,12 @@
      set(ZLIB_DLL_SRCS ${CMAKE_CURRENT_BINARY_DIR}/zlib1rc.obj)
  endif(MINGW)
- 
+
 -add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
++add_library(zlib ${ZLIB_SRCS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
+ target_include_directories(zlib PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
 -add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
+-target_include_directories(zlibstatic PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
 -set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL)
 -set_target_properties(zlib PROPERTIES SOVERSION 1)
-+add_library(zlib ${ZLIB_SRCS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
 +if(BUILD_SHARED_LIBS)
 +    set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL)
 +    set_target_properties(zlib PROPERTIES SOVERSION 1)
 +endif()
- 
+
  if(NOT CYGWIN)
      # This property causes shared libraries on Linux to have the full version
-@@ -165,7 +166,7 @@
- 
+@@ -169,7 +169,7 @@
+
  if(UNIX)
      # On unix-like platforms the library is almost always called libz
 -   set_target_properties(zlib zlibstatic PROPERTIES OUTPUT_NAME z)
 +   set_target_properties(zlib PROPERTIES OUTPUT_NAME z)
-    if(NOT APPLE)
+    if(NOT APPLE AND NOT(CMAKE_SYSTEM_NAME STREQUAL AIX))
       set_target_properties(zlib PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"")
     endif()
-@@ -175,7 +176,7 @@
+@@ -179,7 +179,7 @@
  endif()
- 
+
  if(NOT SKIP_INSTALL_LIBRARIES AND NOT SKIP_INSTALL_ALL )
 -    install(TARGETS zlib zlibstatic
 +    install(TARGETS zlib

depends/common/zlib/02-disable-example-binaries.patch

@@ -1,28 +1,28 @@
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -226,25 +226,3 @@ endif()
- if(NOT SKIP_INSTALL_FILES AND NOT SKIP_INSTALL_ALL )
+@@ -194,25 +194,3 @@ if(NOT SKIP_INSTALL_FILES AND NOT SKIP_INSTALL_ALL )
      install(FILES ${ZLIB_PC} DESTINATION "${INSTALL_PKGCONFIG_DIR}")
  endif()
--
+
 -#============================================================================
 -# Example binaries
 -#============================================================================
+-if(ZLIB_BUILD_EXAMPLES)
+-    add_executable(example test/example.c)
+-    target_link_libraries(example zlib)
+-    add_test(example example)
 -
--add_executable(example test/example.c)
--target_link_libraries(example zlib)
--add_test(example example)
--
--add_executable(minigzip test/minigzip.c)
--target_link_libraries(minigzip zlib)
+-    add_executable(minigzip test/minigzip.c)
+-    target_link_libraries(minigzip zlib)
 -
--if(HAVE_OFF64_T)
--    add_executable(example64 test/example.c)
--    target_link_libraries(example64 zlib)
--    set_target_properties(example64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64")
--    add_test(example64 example64)
+-    if(HAVE_OFF64_T)
+-        add_executable(example64 test/example.c)
+-        target_link_libraries(example64 zlib)
+-        set_target_properties(example64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64")
+-        add_test(example64 example64)
 -
--    add_executable(minigzip64 test/minigzip.c)
--    target_link_libraries(minigzip64 zlib)
--    set_target_properties(minigzip64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64")
+-        add_executable(minigzip64 test/minigzip.c)
+-        target_link_libraries(minigzip64 zlib)
+-        set_target_properties(minigzip64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64")
+-    endif()
 -endif()

depends/common/zlib/03-install-pkgconfig-in-lib.patch

@@ -1,6 +1,6 @@
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -12,7 +12,7 @@ set(INSTALL_BIN_DIR "${CMAKE_INSTALL_PREFIX}/bin" CACHE PATH "Installation direc
+@@ -11,7 +11,7 @@
  set(INSTALL_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib" CACHE PATH "Installation directory for libraries")
  set(INSTALL_INC_DIR "${CMAKE_INSTALL_PREFIX}/include" CACHE PATH "Installation directory for headers")
  set(INSTALL_MAN_DIR "${CMAKE_INSTALL_PREFIX}/share/man" CACHE PATH "Installation directory for manual pages")
@@ -9,4 +9,3 @@
 
  include(CheckTypeSize)
  include(CheckFunctionExists)
-

depends/common/zlib/zlib.sha256

@@ -1 +1 @@
-d14c38e313afc35a9a8760dadf26042f51ea0f5d154b0630a31da0540107fb98
+38ef96b8dfe510d42707d9c781877914792541133e1870841463bfa73f883e32

depends/common/zlib/zlib.txt

@@ -1 +1 @@
-zlib http://mirrors.kodi.tv/build-deps/sources/zlib-1.2.13.tar.xz
+zlib https://www.zlib.net/zlib-1.3.1.tar.xz

pvr.iptvsimple/addon.xml.in

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <addon
   id="pvr.iptvsimple"
-  version="21.10.2"
+  version="21.11.0"
   name="IPTV Simple Client"
   provider-name="nightik and Ross Nicholson">
   <requires>@ADDON_DEPENDS@

pvr.iptvsimple/changelog.txt

@@ -1,3 +1,7 @@
+v21.11.0
+- Add support for web stream extraction from HTML content
+- Add support for custom regex patterns for web stream extraction
+
 v21.10.2
 - Add debug logging for EPG loads
 

src/IptvSimple.cpp

@@ -247,6 +247,7 @@ PVR_ERROR IptvSimple::GetChannelStreamProperties(const kodi::addon::PVRChannel&
     else
       streamURL = m_catchupController.ProcessStreamUrl(m_currentChannel);
 
+    streamURL = StreamUtils::WebStreamExtractor(streamURL, m_currentChannel);
     StreamUtils::SetAllStreamProperties(properties, m_currentChannel, streamURL, catchupUrl.empty(), catchupProperties, m_settings);
 
     Logger::Log(LogLevel::LEVEL_INFO, "%s - Live %s URL: %s", __FUNCTION__, catchupUrl.empty() ? "Stream" : "Catchup", WebUtils::RedactUrl(streamURL).c_str());
@@ -425,6 +426,7 @@ PVR_ERROR IptvSimple::GetRecordingStreamProperties(const kodi::addon::PVRRecordi
 
   if (!mediaEntry.GetMediaEntryId().empty() && !url.empty())
   {
+    url = StreamUtils::WebStreamExtractor(url, mediaEntry);
     StreamUtils::SetAllStreamProperties(properties, mediaEntry, url, m_settings);
 
     return PVR_ERROR_NO_ERROR;