using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; using System.Text.RegularExpressions; using System.Linq; using CodeHollow.FeedReader; using Kruzya.TelegramBot.Core.Extensions; using Kruzya.TelegramBot.RichSiteSummary.Data; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Telegram.Bot.Types; using Telegram.Bot.Types.Enums; using Feed = Kruzya.TelegramBot.RichSiteSummary.Data.Feed; using CodeHollow.FeedReader.Feeds; using Kruzya.TelegramBot.Core.Service; namespace Kruzya.TelegramBot.RichSiteSummary.Service; /// /// Performs a RSS feed parsing. /// Grabs the all feeds from database. /// /// TODO: move entity grabbing to another service. /// public class RssFetch : IHostedService, IDisposable { private Timer _timer; private readonly ILogger _logger; private readonly IServiceScopeFactory _scopeFactory; private readonly ConcurrentQueue _queue; private readonly Hash _hash; private static TimeSpan TimerPeriod => TimeSpan.FromSeconds(45); public RssFetch(ILogger logger, ConcurrentQueue queue, IServiceScopeFactory scopeFactory, Hash hash) { _scopeFactory = scopeFactory; _logger = logger; _queue = queue; _hash = hash; } #region IHostedService /// /// Initializes the RSS fetcher timer. /// /// /// public Task StartAsync(CancellationToken cancellationToken) { _logger.LogInformation("RSS fetcher service is starting."); _timer = new Timer(DoFetch, null, TimeSpan.Zero, TimerPeriod); return Task.CompletedTask; } /// /// Stops the RSS fetcher timer. /// /// /// public Task StopAsync(CancellationToken cancellationToken) { _logger.LogInformation("RSS fetcher service is stopping."); _timer?.Change(Timeout.Infinite, 0); return Task.CompletedTask; } #endregion #region IDisposable /// /// Disposes the timer. /// public void Dispose() { _timer?.Dispose(); } #endregion /// /// Performs the job of fetching RSS data. /// /// private async void DoFetch(object state) { _timer.Change(Timeout.Infinite, 0); _logger.LogDebug("RSS fetcher service is triggered."); try { using var scope = _scopeFactory.CreateScope(); var dbContext = scope.ServiceProvider.GetRequiredService(); var period = scope.ServiceProvider.GetRequiredService() .GetValue("rssFetchPeriod"); var feeds = await dbContext.Feeds.ForFetching(period); _logger.LogDebug("Received {count} feeds for fetching", new {count = feeds.Length}); foreach (var feed in feeds) { await ProcessFeed(feed, dbContext); feed.UpdatedAt = DateTime.Now; dbContext.Update(feed); } await dbContext.SaveChangesAsync(); } finally { _timer.Change(TimerPeriod, TimerPeriod); } } #region Feeds private async Task ProcessFeed(Feed feed, RichSiteSummaryContext dbContext) { var parsedFeed = await FetchFeed(feed); if (parsedFeed == null) { return; } Post feedPost; var newPosts = new List(); var postsImages = new Dictionary(); foreach (var post in parsedFeed.Items) { feedPost = await dbContext.Posts.ByFeedAndUrlHash(_hash.GenerateHash(post.Link), feed); if (feedPost != null) { // skip. This post already exists. continue; } feedPost = dbContext.Posts.Create(); feedPost.Feed = feed; feedPost.Title = post.Title; feedPost.Url = post.Link; feedPost.PostedAt = post.PublishingDate.GetValueOrDefault(DateTime.Now); feedPost.UrlHash = _hash.GenerateHash(feedPost.Url); postsImages.Add(feedPost, await FetchImageUrl(post)); newPosts.Add(feedPost); } if (newPosts.Count > 0) { var subscribers = await dbContext.Subscriptions.ByFeed(feed); foreach (var post in newPosts) { var text = post.MessageText; foreach (var subscriber in subscribers) { var message = new UserMessage { ChatId = new ChatId(subscriber.SubscriberId), DisableWebPagePreview = true, ParseMode = ParseMode.Html, Text = text, ImageUrl = postsImages.GetValueOrDefault(post) }; _queue.Enqueue(message); _logger.LogDebug("Queued message for {SubscriberId}", subscriber.SubscriberId); } } } } private async Task FetchFeed(Feed feed) { try { return await FeedReader.ReadAsync(feed.Url); } catch (Exception e) { _logger.LogError("Feed {feed} can't be fetched: {error}", feed, e.Message); return null; } } #endregion #region Image parsing private async Task FetchImageUrl(FeedItem item) { var feedItem = item.SpecificItem; if (feedItem is Rss20FeedItem rss20FeedItem) { var enclosure = rss20FeedItem.Enclosure; if (enclosure != null && IsValidImage(enclosure.MediaType)) { return enclosure.Url; } } return await FetchImageUrl(item.Description); } private async Task FetchImageUrl(string description) { if (string.IsNullOrWhiteSpace(description)) { return ""; } using var httpClientHandler = new HttpClientHandler() { AllowAutoRedirect = true }; using var httpClient = new HttpClient(httpClientHandler); var imgRegEx = new Regex(@"]+>"); var srcRegEx = new Regex("src=\"([^\"]+)\""); var matches = imgRegEx.Matches(description); var img = ""; foreach (Match match in matches) { try { var srcData = srcRegEx.Match(match.Value); var content = srcData.Groups[1].Value; var response = await httpClient.SendAsync(new HttpRequestMessage(HttpMethod.Head, content)); if (response.StatusCode == HttpStatusCode.OK && IsValidImage(response.Content.Headers.ContentType.MediaType)) { img = content; break; } } catch (Exception e) { _logger.LogError("Caused error when fetching image for RSS post: {error}", e.Message); } } return img; } /// /// Checks if image is valid for Telegram Bot API. /// /// Content-type for received content /// True, if Telegram maybe can "use" this image, false if not. private static bool IsValidImage(string contentType) { return new[] { "image/jpeg", "image/bmp", "image/png" }.Contains(contentType); } #endregion }