telegram-bot/modules/RichSiteSummary/Service/RssFetch.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
using System.Text.RegularExpressions;
using System.Linq;
using CodeHollow.FeedReader;
using Kruzya.TelegramBot.Core.Extensions;
using Kruzya.TelegramBot.RichSiteSummary.Data;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Telegram.Bot.Types;
using Telegram.Bot.Types.Enums;
using Feed = Kruzya.TelegramBot.RichSiteSummary.Data.Feed;

using CodeHollow.FeedReader.Feeds;
using CodeHollow.FeedReader.Feeds.Itunes;
using CodeHollow.FeedReader.Feeds.MediaRSS;

namespace Kruzya.TelegramBot.RichSiteSummary.Service
{
    /// <summary>
    /// Performs a RSS feed parsing.
    /// Grabs the all feeds from database.
    ///
    /// TODO: move entity grabbing to another service.
    /// </summary>
    public class RssFetch : IHostedService, IDisposable
    {
        private Timer _timer;
        
        private readonly ILogger _logger;
        private readonly IServiceScopeFactory _scopeFactory;
        private readonly ConcurrentQueue<UserMessage> _queue;

        private TimeSpan timerPeriod => TimeSpan.FromSeconds(45);

        public RssFetch(ILogger<RssFetch> logger, ConcurrentQueue<UserMessage> queue, IServiceScopeFactory scopeFactory)
        {
            _scopeFactory = scopeFactory;
            _logger = logger;
            _queue = queue;
        }

        #region IHostedService
        /// <summary>
        /// Initializes the RSS fetcher timer.
        /// </summary>
        /// <param name="cancellationToken"></param>
        /// <returns></returns>
        public Task StartAsync(CancellationToken cancellationToken)
        {
            _logger.LogInformation("RSS fetcher service is starting.");
            _timer = new Timer(DoFetch, null, TimeSpan.Zero, timerPeriod);

            return Task.CompletedTask;
        }

        /// <summary>
        /// Stops the RSS fetcher timer.
        /// </summary>
        /// <param name="cancellationToken"></param>
        /// <returns></returns>
        public Task StopAsync(CancellationToken cancellationToken)
        {
            _logger.LogInformation("RSS fetcher service is stopping.");
            _timer?.Change(Timeout.Infinite, 0);

            return Task.CompletedTask;
        }
        #endregion
        #region IDisposable
        /// <summary>
        /// Disposes the timer.
        /// </summary>
        public void Dispose()
        {
            _timer?.Dispose();
        }
        #endregion

        /// <summary>
        /// Performs the job of fetching RSS data.
        /// </summary>
        /// <param name="state"></param>
        private async void DoFetch(object state)
        {
            _timer.Change(Timeout.Infinite, 0);
            _logger.LogDebug("RSS fetcher service is triggered.");

            try
            {
                using var scope = _scopeFactory.CreateScope();
                var dbContext = scope.ServiceProvider.GetRequiredService<RichSiteSummaryContext>();
                var period = scope.ServiceProvider.GetRequiredService<IConfiguration>()
                    .GetValue<UInt16>("rssFetchPeriod");

                var feeds = await dbContext.Feeds.ForFetching(period);
                _logger.LogDebug("Received {count} feeds for fetching", new {count = feeds.Length});

                foreach (var feed in feeds)
                {
                    await ProcessFeed(feed, dbContext);

                    feed.UpdatedAt = DateTime.Now;
                    dbContext.Update(feed);
                }

                await dbContext.SaveChangesAsync();
            }
            finally
            {
                _timer.Change(timerPeriod, timerPeriod);
            }
        }

        #region Feeds

        private async Task ProcessFeed(Feed feed, RichSiteSummaryContext dbContext)
        {
            var parsedFeed = await FetchFeed(feed);
            if (parsedFeed == null)
            {
                return;
            }
            
            Post feedPost;
            var newPosts = new List<Post>();
            var postsImages = new Dictionary<Post, string>();
            foreach (var post in parsedFeed.Items)
            {
                feedPost = await dbContext.Posts.ByFeedAndUrl(post.Link, feed);
                if (feedPost != null)
                {
                    // skip. This post already exists.
                    continue;
                }

                feedPost = dbContext.Posts.Create();
                feedPost.Feed = feed;
                feedPost.Title = post.Title;
                feedPost.Url = post.Link;
                feedPost.PostedAt = post.PublishingDate.GetValueOrDefault(DateTime.Now);

                postsImages.Add(feedPost, await FetchImageUrl(post));
                newPosts.Add(feedPost);
            }

            if (newPosts.Count > 0)
            {
                var subscribers = await dbContext.Subscriptions.ByFeed(feed);
                foreach (var post in newPosts)
                {
                    var text = post.MessageText;
                    foreach (var subscriber in subscribers)
                    {
                        var message = new UserMessage()
                        {
                            ChatId = new ChatId(subscriber.SubscriberId), DisableWebPagePreview = true,
                            ParseMode = ParseMode.Html, Text = text, ImageUrl = postsImages.GetValueOrDefault(post)
                        };

                        _queue.Enqueue(message);
                        _logger.LogDebug($"Enqueued message for {subscriber.SubscriberId}");
                    }
                }
            }
        }

        private async Task<CodeHollow.FeedReader.Feed> FetchFeed(Feed feed)
        {
            try
            {
                return await FeedReader.ReadAsync(feed.Url.ToString());
            }
            catch (Exception e)
            {
                _logger.LogError($"Feed {feed} can't be fetched: {e.Message}");
                return null;
            }
        }

        #endregion

        #region Image parsing

        private async Task<string> FetchImageUrl(FeedItem item)
        {
            var feedItem = item.SpecificItem;
            if (feedItem is Rss20FeedItem)
            {
                var enclosure = ((Rss20FeedItem)feedItem).Enclosure;
                if (enclosure != null && IsValidImage(enclosure.MediaType))
                {
                    return enclosure.Url;
                }
            }

            return await FetchImageUrl(item.Description);
        }

        private async Task<string> FetchImageUrl(string description)
        {
            if (string.IsNullOrWhiteSpace(description))
            {
                return "";
            }

            using var httpClientHandler = new HttpClientHandler() { AllowAutoRedirect = true };
            using var httpClient = new HttpClient(httpClientHandler);

            var imgRegEx = new Regex(@"<img[^>]+>");
            var srcRegEx = new Regex("src=\"([^\"]+)\"");
            var matches = imgRegEx.Matches(description);

            var img = "";
            foreach (Match match in matches)
            {
                try
                {
                    var srcData = srcRegEx.Match(match.Value);
                    var content = srcData.Groups[1].Value;

                    var response = await httpClient.SendAsync(new HttpRequestMessage(HttpMethod.Head, content));
                    if (response.StatusCode == HttpStatusCode.OK &&
                        IsValidImage(response.Content.Headers.ContentType.MediaType))
                    {
                        img = content;
                        break;
                    }
                }
                catch (Exception e)
                {
                    _logger.LogError($"Caused error when fetching image for RSS post: {e.Message}");
                }
            }

            return img;
        }

        /// <summary>
        /// Checks if image is valid for Telegram Bot API.
        /// </summary>
        /// <param name="contentType">Content-type for received content</param>
        /// <returns>True, if Telegram maybe can "use" this image, false if not.</returns>
        private static bool IsValidImage(string contentType)
        {
            return new string[]
            {
                "image/jpeg",
                "image/bmp",
                "image/png"
            }.Contains(contentType);
        }

        #endregion
    }
}