hamkadr/src/JobsMedical.Web/Services/Scraping/IngestionWorker.cs

namespace JobsMedical.Web.Services.Scraping;

/// <summary>
/// Periodically runs the ingestion engine when the admin has turned auto-ingest ON
/// (AppSetting.AutoIngestEnabled) — read fresh from the DB each cycle, so it can be toggled at
/// runtime from the admin panel with no redeploy. When off, it idles and re-checks.
/// </summary>
public class IngestionWorker : BackgroundService
{
    private readonly IServiceScopeFactory _scopes;
    private readonly ILogger<IngestionWorker> _log;

    public IngestionWorker(IServiceScopeFactory scopes, ILogger<IngestionWorker> log)
    {
        _scopes = scopes;
        _log = log;
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        // Small startup delay so the DB/migrations are ready.
        try { await Task.Delay(TimeSpan.FromSeconds(20), stoppingToken); }
        catch (OperationCanceledException) { return; }

        while (!stoppingToken.IsCancellationRequested)
        {
            var idleMinutes = 10;
            try
            {
                using var scope = _scopes.CreateScope();

                // Always archive stale listings (independent of ingestion being on).
                await scope.ServiceProvider.GetRequiredService<ListingArchiver>().ArchiveStaleAsync(stoppingToken);

                var settings = await scope.ServiceProvider
                    .GetRequiredService<SettingsService>().GetAsync();

                if (settings.AutoIngestEnabled)
                {
                    var svc = scope.ServiceProvider.GetRequiredService<IngestionService>();
                    var summary = await svc.RunAsync(stoppingToken);
                    _log.LogInformation("Auto-ingest: queued={Q} published={P} flagged={F} spam={S} dupes={D}",
                        summary.TotalQueued, summary.TotalPublished, summary.TotalFlagged,
                        summary.TotalSpam, summary.TotalDuplicates);
                    idleMinutes = Math.Max(1, settings.IngestIntervalMinutes);
                }
            }
            catch (Exception ex) when (ex is not OperationCanceledException)
            {
                _log.LogError(ex, "Auto-ingest cycle failed");
            }

            try { await Task.Delay(TimeSpan.FromMinutes(idleMinutes), stoppingToken); }
            catch (OperationCanceledException) { break; }
        }
    }
}